From c3e392f87003282af2b631b3eb581877f0d0f25d Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 27 May 2021 15:09:31 -0400 Subject: [PATCH 001/225] Map PageRevision declaratively --- lib/galaxy/model/__init__.py | 14 +++++++++++++- lib/galaxy/model/mapping.py | 16 ++-------------- test/unit/model/test_model.py | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ae3d3fcbdee..7a08fff2f9e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -37,6 +37,7 @@ from sqlalchemy import ( Boolean, Column, DateTime, + ForeignKey, func, inspect, Integer, @@ -72,6 +73,7 @@ import galaxy.model.orm.now import galaxy.model.tags import galaxy.security.passwords import galaxy.util +from galaxy.model.custom_types import TrimmedString from galaxy.model.item_attrs import get_item_annotation_str, UsesAnnotations from galaxy.model.orm.now import now from galaxy.security import get_permitted_actions @@ -6451,7 +6453,17 @@ class Page(Dictifiable, RepresentById): return self.user.username -class PageRevision(Dictifiable, RepresentById): +class PageRevision(Base, Dictifiable, RepresentById): + __tablename__ = 'page_revision' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + page_id = Column('page_id', Integer, ForeignKey('page.id'), index=True, nullable=False) + title = Column('title', TEXT) + content = Column('content', TEXT) + content_format = Column('content_format', TrimmedString(32)) + DEFAULT_CONTENT_FORMAT = 'html' dict_element_visible_keys = ['id', 'page_id', 'title', 'content', 'content_format'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 9357e1800b9..9cc415f4d26 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1193,16 +1193,6 @@ model.Page.table = Table( Index('ix_page_slug', 'slug', mysql_length=200), ) -model.PageRevision.table = Table( - "page_revision", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("page_id", Integer, ForeignKey("page.id"), index=True, nullable=False), - Column("title", TEXT), - Column("content", TEXT), - Column("content_format", TrimmedString(32))) - model.PageUserShareAssociation.table = Table( "page_user_share_association", metadata, Column("id", Integer, primary_key=True), @@ -2521,17 +2511,15 @@ simple_mapping( ) -mapper_registry.map_imperatively(model.PageRevision, model.PageRevision.table) - mapper_registry.map_imperatively(model.Page, model.Page.table, properties=dict( user=relation(model.User), revisions=relation(model.PageRevision, backref='page', cascade="all, delete-orphan", - primaryjoin=(model.Page.table.c.id == model.PageRevision.table.c.page_id)), + primaryjoin=(model.Page.table.c.id == model.PageRevision.page_id)), latest_revision=relation(model.PageRevision, post_update=True, - primaryjoin=(model.Page.table.c.latest_revision_id == model.PageRevision.table.c.id), + primaryjoin=(model.Page.table.c.latest_revision_id == model.PageRevision.id), lazy=False), tags=relation(model.PageTagAssociation, order_by=model.PageTagAssociation.table.c.id, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 7ca0b8650c8..46e23277d1a 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -21,6 +21,18 @@ def session(model): Session.remove() # Ensures we get a new session for each test +@pytest.fixture +def user(model): + return model.User(email='test@example.com', password='password') + + +@pytest.fixture +def page(model, user): + p = model.Page() + p.user = user + return p + + def test_Group_table(model): tbl = model.Group.__table__ assert tbl.name == 'galaxy_group' @@ -43,6 +55,30 @@ def test_Group(model, session): cleanup(session, cls) +def test_PageRevision_table(model): + tbl = model.PageRevision.__table__ + assert tbl.name == 'page_revision' + + +def test_PageRevision(model, session, page): + cls = model.PageRevision + obj = cls() + obj.page = page + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.page_id + assert stored_obj.title is None + assert stored_obj.content is None + assert stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT + + cleanup(session, cls) + + def test_Quota_table(model): tbl = model.Quota.__table__ assert tbl.name == 'quota' From e186e905280f97e2f2418f5a7343793c32d780b8 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 27 May 2021 15:20:37 -0400 Subject: [PATCH 002/225] Do not initialize instrumented attributes to None With declarative mapping this is redundant. --- lib/galaxy/model/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7a08fff2f9e..049c0f8327c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -293,7 +293,6 @@ class WorkerProcess(Base, UsesCreateAndUpdateTime, _HasTable): def __init__(self, server_name, hostname): self.server_name = server_name self.hostname = hostname - self.pid = None def cached_id(galaxy_model_object): @@ -6468,9 +6467,6 @@ class PageRevision(Base, Dictifiable, RepresentById): dict_element_visible_keys = ['id', 'page_id', 'title', 'content', 'content_format'] def __init__(self): - self.user = None - self.title = None - self.content = None self.content_format = PageRevision.DEFAULT_CONTENT_FORMAT def to_dict(self, view='element'): From 7c4bebe209cf08bd9797f7111523366e0aaf403b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 27 May 2021 16:02:43 -0400 Subject: [PATCH 003/225] Map VisualizationRevision declaratively --- lib/galaxy/model/__init__.py | 22 ++++++++++++++++++-- lib/galaxy/model/mapping.py | 18 ++-------------- test/unit/model/test_model.py | 39 +++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 049c0f8327c..b51da8f5c03 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -39,6 +39,7 @@ from sqlalchemy import ( DateTime, ForeignKey, func, + Index, inspect, Integer, join, @@ -73,7 +74,10 @@ import galaxy.model.orm.now import galaxy.model.tags import galaxy.security.passwords import galaxy.util -from galaxy.model.custom_types import TrimmedString +from galaxy.model.custom_types import ( + MutableJSONType, + TrimmedString, +) from galaxy.model.item_attrs import get_item_annotation_str, UsesAnnotations from galaxy.model.orm.now import now from galaxy.security import get_permitted_actions @@ -6517,7 +6521,21 @@ class Visualization(RepresentById): return copy_viz -class VisualizationRevision(RepresentById): +class VisualizationRevision(Base, RepresentById): + __tablename__ = 'visualization_revision' + __table_args__ = ( + Index('ix_visualization_revision_dbkey', 'dbkey', mysql_length=200), + ) + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + visualization_id = Column('visualization_id', Integer, ForeignKey('visualization.id'), + index=True, nullable=False) + title = Column('title', TEXT) + dbkey = Column('dbkey', TEXT) + config = Column('config', MutableJSONType) + def __init__(self, visualization=None, title=None, dbkey=None, config=None): self.id = None self.visualization = visualization diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 9cc415f4d26..ace111274d9 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1218,18 +1218,6 @@ model.Visualization.table = Table( Index('ix_visualization_slug', 'slug', mysql_length=200), ) -model.VisualizationRevision.table = Table( - "visualization_revision", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("visualization_id", Integer, ForeignKey("visualization.id"), index=True, nullable=False), - Column("title", TEXT), - Column("dbkey", TEXT), - Column("config", MutableJSONType), - Index('ix_visualization_revision_dbkey', 'dbkey', mysql_length=200), -) - model.VisualizationUserShareAssociation.table = Table( "visualization_user_share_association", metadata, Column("id", Integer, primary_key=True), @@ -2545,17 +2533,15 @@ mapper_registry.map_imperatively(model.PageUserShareAssociation, model.PageUserS properties=dict(user=relation(model.User, backref='pages_shared_by_others'), page=relation(model.Page, backref='users_shared_with'))) -mapper_registry.map_imperatively(model.VisualizationRevision, model.VisualizationRevision.table) - mapper_registry.map_imperatively(model.Visualization, model.Visualization.table, properties=dict( user=relation(model.User), revisions=relation(model.VisualizationRevision, backref='visualization', cascade="all, delete-orphan", - primaryjoin=(model.Visualization.table.c.id == model.VisualizationRevision.table.c.visualization_id)), + primaryjoin=(model.Visualization.table.c.id == model.VisualizationRevision.visualization_id)), latest_revision=relation(model.VisualizationRevision, post_update=True, - primaryjoin=(model.Visualization.table.c.latest_revision_id == model.VisualizationRevision.table.c.id), + primaryjoin=(model.Visualization.table.c.latest_revision_id == model.VisualizationRevision.id), lazy=False), tags=relation(model.VisualizationTagAssociation, order_by=model.VisualizationTagAssociation.table.c.id, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 46e23277d1a..915597ec4f8 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -33,6 +33,13 @@ def page(model, user): return p +@pytest.fixture +def visualization(model, user): + v = model.Visualization() + v.user = user + return v + + def test_Group_table(model): tbl = model.Group.__table__ assert tbl.name == 'galaxy_group' @@ -239,6 +246,31 @@ def test_PSAPartial(model, session): cleanup(session, cls) +def test_VisualizationRevision_table(model): + tbl = model.VisualizationRevision.__table__ + assert tbl.name == 'visualization_revision' + assert has_index(tbl, ('dbkey',)) + + +def test_VisualizationRevision(model, session, visualization): + cls = model.VisualizationRevision + obj = cls() + obj.visualization = visualization + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.visualization_id + assert stored_obj.title is None + assert stored_obj.dbkey is None + assert stored_obj.config is None + + cleanup(session, cls) + + def persist(session, obj): session.add(obj) session.flush() @@ -254,3 +286,10 @@ def has_unique_constraint(table, fields): col_names = {c.name for c in constraint.columns} if set(fields) == col_names: return True + + +def has_index(table, fields): + for index in table.indexes: + col_names = {c.name for c in index.columns} + if set(fields) == col_names: + return True From 03484f664d050363fa762eff4a9d8ae99c93f19a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 28 May 2021 02:05:54 -0400 Subject: [PATCH 004/225] Map UserAddress declaratively (see note) 1. Use back_populates instead of backref. We specify both sides of the relationship (e.g. on both parent and child); this makes the model more explicit and much easier to comprehend. 2. Do not specify a default primaryjoin on a relationship. 3. In unit tests, call session.expunge() following flush in persist() function: this ensures that we get a new object on subsequent load from the db, which preserves the test validity. --- lib/galaxy/model/__init__.py | 25 +++++++++++++++++++++++- lib/galaxy/model/mapping.py | 30 +++-------------------------- test/unit/model/test_model.py | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 28 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b51da8f5c03..3eab54810c4 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -28,6 +28,7 @@ from typing import ( ) from uuid import UUID, uuid4 +import sqlalchemy from boltons.iterutils import remap from social_core.storage import AssociationMixin, CodeMixin, NonceMixin, PartialMixin, UserMixin from sqlalchemy import ( @@ -65,6 +66,7 @@ from sqlalchemy.orm import ( Query, reconstructor, registry, + relationship, ) from sqlalchemy.orm.decl_api import DeclarativeMeta @@ -6121,7 +6123,28 @@ class FormValues(RepresentById): self.content = content -class UserAddress(RepresentById): +class UserAddress(Base, RepresentById): + __tablename__ = 'user_address' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + desc = Column('desc', TrimmedString(255)) + name = Column('name', TrimmedString(255), nullable=False) + institution = Column('institution', TrimmedString(255)) + address = Column('address', TrimmedString(255), nullable=False) + city = Column('city', TrimmedString(255), nullable=False) + state = Column('state', TrimmedString(255), nullable=False) + postal_code = Column('postal_code', TrimmedString(255), nullable=False) + country = Column('country', TrimmedString(255), nullable=False) + phone = Column('phone', TrimmedString(255)) + deleted = Column('deleted', Boolean, index=True, default=False) + purged = Column('purged', Boolean, index=True, default=False) + # `desc` needs to be fully qualified because it is shadowed by `desc` Column defined above + # TODO: db migration to rename column, then use `desc` + user = relationship('User', back_populates='addresses', order_by=sqlalchemy.desc('update_time')) + def __init__(self, user=None, desc=None, name=None, institution=None, address=None, city=None, state=None, postal_code=None, country=None, phone=None): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index ace111274d9..705e6319104 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -76,24 +76,6 @@ model.User.table = Table( Column("active", Boolean, index=True, default=True, nullable=False), Column("activation_token", TrimmedString(64), nullable=True, index=True)) -model.UserAddress.table = Table( - "user_address", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("desc", TrimmedString(255)), - Column("name", TrimmedString(255), nullable=False), - Column("institution", TrimmedString(255)), - Column("address", TrimmedString(255), nullable=False), - Column("city", TrimmedString(255), nullable=False), - Column("state", TrimmedString(255), nullable=False), - Column("postal_code", TrimmedString(255), nullable=False), - Column("country", TrimmedString(255), nullable=False), - Column("phone", TrimmedString(255)), - Column("deleted", Boolean, index=True, default=False), - Column("purged", Boolean, index=True, default=False)) - model.UserAuthnzToken.table = Table( "oidc_user_authnz_tokens", metadata, Column('id', Integer, primary_key=True), @@ -1598,13 +1580,6 @@ mapper_registry.map_imperatively(model.FormDefinitionCurrent, model.FormDefiniti primaryjoin=(model.FormDefinitionCurrent.table.c.latest_form_id == model.FormDefinition.table.c.id)) )) -mapper_registry.map_imperatively(model.UserAddress, model.UserAddress.table, properties=dict( - user=relation(model.User, - primaryjoin=(model.UserAddress.table.c.user_id == model.User.table.c.id), - backref='addresses', - order_by=desc(model.UserAddress.table.c.update_time)), -)) - mapper_registry.map_imperatively(model.UserAuthnzToken, model.UserAuthnzToken.table, properties=dict( user=relation(model.User, primaryjoin=(model.UserAuthnzToken.table.c.user_id == model.User.table.c.id), @@ -1798,6 +1773,9 @@ mapper_registry.map_imperatively(model.HistoryUserShareAssociation, model.Histor )) mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( + addresses=relation(model.UserAddress, + back_populates='user', + order_by=desc(model.UserAddress.update_time)), histories=relation(model.History, backref="user", order_by=desc(model.History.update_time)), @@ -1823,8 +1801,6 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( _preferences=relation(model.UserPreference, backref="user", collection_class=attribute_mapped_collection('name')), - # addresses=relation( UserAddress, - # primaryjoin=( User.table.c.id == UserAddress.table.c.user_id ) ), values=relation(model.FormValues, primaryjoin=(model.User.table.c.form_values_id == model.FormValues.table.c.id)), api_keys=relation(model.APIKeys, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 915597ec4f8..70f84a9db36 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -246,6 +246,40 @@ def test_PSAPartial(model, session): cleanup(session, cls) +def test_UserAddress_table(model): + tbl = model.UserAddress.__table__ + assert tbl.name == 'user_address' + + +def test_UserAddress(model, session, user): + cls = model.UserAddress + desc, name, institution, address, city, state, postal_code, country, phone = \ + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i' + obj = cls(user, desc, name, institution, address, city, state, postal_code, country, phone) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.user_id == user.id + assert stored_obj.desc == desc + assert stored_obj.name == name + assert stored_obj.institution == institution + assert stored_obj.address == address + assert stored_obj.city == city + assert stored_obj.state == state + assert stored_obj.postal_code == postal_code + assert stored_obj.country == country + assert stored_obj.phone == phone + assert stored_obj.deleted is False + assert stored_obj.purged is False + assert stored_obj.user == user + + cleanup(session, cls) + + def test_VisualizationRevision_table(model): tbl = model.VisualizationRevision.__table__ assert tbl.name == 'visualization_revision' @@ -274,6 +308,8 @@ def test_VisualizationRevision(model, session, visualization): def persist(session, obj): session.add(obj) session.flush() + # Remove from session, so that on a subsequent load we get a *new* obj from the db + session.expunge(obj) def cleanup(session, cls): From 69341998ff9d54e0ae21ab8320b2195354024902 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 28 May 2021 22:21:16 -0400 Subject: [PATCH 005/225] Refactor tests; ensure db cleanup after each test --- test/unit/model/test_model.py | 499 ++++++++++++++++------------------ 1 file changed, 228 insertions(+), 271 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 70f84a9db36..dc14895e408 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1,3 +1,5 @@ +from contextlib import contextmanager + import pytest from sqlalchemy import ( delete, @@ -8,6 +10,213 @@ from sqlalchemy import ( import galaxy.model.mapping as mapping +def test_Group(model, session): + cls = model.Group + assert cls.__tablename__ == 'galaxy_group' + with dbcleanup(session, cls): + name = 'a' + obj = cls(name) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.name == name + assert stored_obj.deleted is False + + +def test_PageRevision(model, session, page): + cls = model.PageRevision + assert cls.__tablename__ == 'page_revision' + with dbcleanup(session, cls): + obj = cls() + obj.page = page + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.page_id + assert stored_obj.title is None + assert stored_obj.content is None + assert stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT + + +def test_Quota(model, session): + cls = model.Quota + assert cls.__tablename__ == 'quota' + with dbcleanup(session, cls): + name, description = 'a', 'b' + obj = cls(name, description) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.name == name + assert stored_obj.description == description + assert stored_obj.bytes == 0 + assert stored_obj.operation == '=' + assert stored_obj.deleted is False + + +def test_Role(model, session): + cls = model.Role + assert cls.__tablename__ == 'role' + with dbcleanup(session, cls): + name, description = 'a', 'b' + obj = cls(name, description) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.name == name + assert stored_obj.description == description + assert stored_obj.type == model.Role.types.SYSTEM + assert stored_obj.deleted is False + + +def test_WorkerProcess(model, session): + cls = model.WorkerProcess + assert cls.__tablename__ == 'worker_process' + with dbcleanup(session, cls): + server_name, hostname = 'a', 'b' + obj = cls(server_name, hostname) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.server_name == server_name + assert stored_obj.hostname == hostname + assert stored_obj.pid is None + assert stored_obj.update_time + + +def test_PSAAssociation(model, session): + cls = model.PSAAssociation + assert cls.__tablename__ == 'psa_association' + with dbcleanup(session, cls): + server_url, handle, secret, issued, lifetime, assoc_type = 'a', 'b', 'c', 1, 2, 'd' + obj = cls(server_url, handle, secret, issued, lifetime, assoc_type) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.server_url == server_url + assert stored_obj.handle == handle + assert stored_obj.secret == secret + assert stored_obj.issued == issued + assert stored_obj.lifetime == lifetime + assert stored_obj.assoc_type == assoc_type + + +def test_PSACode(model, session): + cls = model.PSACode + assert cls.__tablename__ == 'psa_code' + with dbcleanup(session, cls): + email, code = 'a', 'b' + obj = cls(email, code) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.email == email + assert stored_obj.code == code + + +def test_PSANonce(model, session): + cls = model.PSANonce + assert cls.__tablename__ == 'psa_nonce' + with dbcleanup(session, cls): + server_url, timestamp, salt = 'a', 1, 'b' + obj = cls(server_url, timestamp, salt) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.server_url + assert stored_obj.timestamp == timestamp + assert stored_obj.salt == salt + + +def test_PSAPartial(model, session): + cls = model.PSAPartial + assert cls.__tablename__ == 'psa_partial' + with dbcleanup(session, cls): + token, data, next_step, backend = 'a', 'b', 1, 'c' + obj = cls(token, data, next_step, backend) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.token == token + assert stored_obj.data == data + assert stored_obj.next_step == next_step + assert stored_obj.backend == backend + + +def test_UserAddress(model, session, user): + cls = model.UserAddress + assert cls.__tablename__ == 'user_address' + with dbcleanup(session, cls): + desc, name, institution, address, city, state, postal_code, country, phone = \ + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i' + obj = cls(user, desc, name, institution, address, city, state, postal_code, country, phone) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.user_id == user.id + assert stored_obj.desc == desc + assert stored_obj.name == name + assert stored_obj.institution == institution + assert stored_obj.address == address + assert stored_obj.city == city + assert stored_obj.state == state + assert stored_obj.postal_code == postal_code + assert stored_obj.country == country + assert stored_obj.phone == phone + assert stored_obj.deleted is False + assert stored_obj.purged is False + assert stored_obj.user == user + + +def test_VisualizationRevision(model, session, visualization): + cls = model.VisualizationRevision + assert cls.__tablename__ == 'visualization_revision' + with dbcleanup(session, cls): + obj = cls() + obj.visualization = visualization + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.visualization_id + assert stored_obj.title is None + assert stored_obj.dbkey is None + assert stored_obj.config is None + + @pytest.fixture(scope='module') def model(): db_uri = 'sqlite:///:memory:' @@ -22,287 +231,39 @@ def session(model): @pytest.fixture -def user(model): - return model.User(email='test@example.com', password='password') +def user(model, session): + u = model.User(email='test@example.com', password='password') + yield from dbcleanup_wrapper(session, u) @pytest.fixture -def page(model, user): +def page(model, session, user): p = model.Page() p.user = user - return p + yield from dbcleanup_wrapper(session, p) @pytest.fixture -def visualization(model, user): +def visualization(model, session, user): v = model.Visualization() v.user = user - return v + yield from dbcleanup_wrapper(session, v) -def test_Group_table(model): - tbl = model.Group.__table__ - assert tbl.name == 'galaxy_group' +@contextmanager +def dbcleanup(session, cls): + """ + Ensure all records of cls are deleted from the database on exit. + """ + try: + yield + finally: + session.execute(delete(cls)) -def test_Group(model, session): - cls = model.Group - name = 'a' - obj = cls(name) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.create_time - assert stored_obj.update_time - assert stored_obj.name == name - assert stored_obj.deleted is False - - cleanup(session, cls) - - -def test_PageRevision_table(model): - tbl = model.PageRevision.__table__ - assert tbl.name == 'page_revision' - - -def test_PageRevision(model, session, page): - cls = model.PageRevision - obj = cls() - obj.page = page - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.create_time - assert stored_obj.update_time - assert stored_obj.page_id - assert stored_obj.title is None - assert stored_obj.content is None - assert stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT - - cleanup(session, cls) - - -def test_Quota_table(model): - tbl = model.Quota.__table__ - assert tbl.name == 'quota' - - -def test_Quota(model, session): - cls = model.Quota - name, description = 'a', 'b' - obj = cls(name, description) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.create_time - assert stored_obj.update_time - assert stored_obj.name == name - assert stored_obj.description == description - assert stored_obj.bytes == 0 - assert stored_obj.operation == '=' - assert stored_obj.deleted is False - - cleanup(session, cls) - - -def test_Role_table(model): - tbl = model.Role.__table__ - assert tbl.name == 'role' - - -def test_Role(model, session): - cls = model.Role - name, description = 'a', 'b' - obj = cls(name, description) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.create_time - assert stored_obj.update_time - assert stored_obj.name == name - assert stored_obj.description == description - assert stored_obj.type == model.Role.types.SYSTEM - assert stored_obj.deleted is False - - cleanup(session, cls) - - -def test_WorkerProcess_table(model): - tbl = model.WorkerProcess.__table__ - assert tbl.name == 'worker_process' - assert has_unique_constraint(tbl, ('server_name', 'hostname')) - - -def test_WorkerProcess(model, session): - cls = model.WorkerProcess - server_name, hostname = 'a', 'b' - obj = cls(server_name, hostname) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.server_name == server_name - assert stored_obj.hostname == hostname - assert stored_obj.pid is None - assert stored_obj.update_time - - cleanup(session, cls) - - -def test_PSAAssociation_table(model): - tbl = model.PSAAssociation.__table__ - assert tbl.name == 'psa_association' - - -def test_PSAAssociation(model, session): - cls = model.PSAAssociation - server_url, handle, secret, issued, lifetime, assoc_type = 'a', 'b', 'c', 1, 2, 'd' - obj = cls(server_url, handle, secret, issued, lifetime, assoc_type) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.server_url == server_url - assert stored_obj.handle == handle - assert stored_obj.secret == secret - assert stored_obj.issued == issued - assert stored_obj.lifetime == lifetime - assert stored_obj.assoc_type == assoc_type - - cleanup(session, cls) - - -def test_PSACode_table(model): - tbl = model.PSACode.__table__ - assert tbl.name == 'psa_code' - assert has_unique_constraint(tbl, ('code', 'email')) - - -def test_PSACode(model, session): - cls = model.PSACode - email, code = 'a', 'b' - obj = cls(email, code) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.email == email - assert stored_obj.code == code - - cleanup(session, cls) - - -def test_PSANonce_table(model): - tbl = model.PSANonce.__table__ - assert tbl.name == 'psa_nonce' - - -def test_PSANonce(model, session): - cls = model.PSANonce - server_url, timestamp, salt = 'a', 1, 'b' - obj = cls(server_url, timestamp, salt) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.server_url - assert stored_obj.timestamp == timestamp - assert stored_obj.salt == salt - - cleanup(session, cls) - - -def test_PSAPartial_table(model): - tbl = model.PSAPartial.__table__ - assert tbl.name == 'psa_partial' - - -def test_PSAPartial(model, session): - cls = model.PSAPartial - token, data, next_step, backend = 'a', 'b', 1, 'c' - obj = cls(token, data, next_step, backend) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.token == token - assert stored_obj.data == data - assert stored_obj.next_step == next_step - assert stored_obj.backend == backend - - cleanup(session, cls) - - -def test_UserAddress_table(model): - tbl = model.UserAddress.__table__ - assert tbl.name == 'user_address' - - -def test_UserAddress(model, session, user): - cls = model.UserAddress - desc, name, institution, address, city, state, postal_code, country, phone = \ - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i' - obj = cls(user, desc, name, institution, address, city, state, postal_code, country, phone) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.create_time - assert stored_obj.update_time - assert stored_obj.user_id == user.id - assert stored_obj.desc == desc - assert stored_obj.name == name - assert stored_obj.institution == institution - assert stored_obj.address == address - assert stored_obj.city == city - assert stored_obj.state == state - assert stored_obj.postal_code == postal_code - assert stored_obj.country == country - assert stored_obj.phone == phone - assert stored_obj.deleted is False - assert stored_obj.purged is False - assert stored_obj.user == user - - cleanup(session, cls) - - -def test_VisualizationRevision_table(model): - tbl = model.VisualizationRevision.__table__ - assert tbl.name == 'visualization_revision' - assert has_index(tbl, ('dbkey',)) - - -def test_VisualizationRevision(model, session, visualization): - cls = model.VisualizationRevision - obj = cls() - obj.visualization = visualization - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.create_time - assert stored_obj.update_time - assert stored_obj.visualization_id - assert stored_obj.title is None - assert stored_obj.dbkey is None - assert stored_obj.config is None - - cleanup(session, cls) +def dbcleanup_wrapper(session, obj): + with dbcleanup(session, type(obj)): + yield obj def persist(session, obj): @@ -312,10 +273,6 @@ def persist(session, obj): session.expunge(obj) -def cleanup(session, cls): - session.execute(delete(cls)) - - def has_unique_constraint(table, fields): for constraint in table.constraints: if isinstance(constraint, UniqueConstraint): From 0227f9f7f898862d5716bfad456526aa2c78524a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 3 Jun 2021 11:21:28 -0400 Subject: [PATCH 006/225] Rearrange tests alphabetically --- test/unit/model/test_model.py | 112 +++++++++++++++++----------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index dc14895e408..6cb7f3bcfd9 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -46,62 +46,6 @@ def test_PageRevision(model, session, page): assert stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT -def test_Quota(model, session): - cls = model.Quota - assert cls.__tablename__ == 'quota' - with dbcleanup(session, cls): - name, description = 'a', 'b' - obj = cls(name, description) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.create_time - assert stored_obj.update_time - assert stored_obj.name == name - assert stored_obj.description == description - assert stored_obj.bytes == 0 - assert stored_obj.operation == '=' - assert stored_obj.deleted is False - - -def test_Role(model, session): - cls = model.Role - assert cls.__tablename__ == 'role' - with dbcleanup(session, cls): - name, description = 'a', 'b' - obj = cls(name, description) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.create_time - assert stored_obj.update_time - assert stored_obj.name == name - assert stored_obj.description == description - assert stored_obj.type == model.Role.types.SYSTEM - assert stored_obj.deleted is False - - -def test_WorkerProcess(model, session): - cls = model.WorkerProcess - assert cls.__tablename__ == 'worker_process' - with dbcleanup(session, cls): - server_name, hostname = 'a', 'b' - obj = cls(server_name, hostname) - persist(session, obj) - - stmt = select(cls) - stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.server_name == server_name - assert stored_obj.hostname == hostname - assert stored_obj.pid is None - assert stored_obj.update_time - - def test_PSAAssociation(model, session): cls = model.PSAAssociation assert cls.__tablename__ == 'psa_association' @@ -169,6 +113,45 @@ def test_PSAPartial(model, session): assert stored_obj.backend == backend +def test_Quota(model, session): + cls = model.Quota + assert cls.__tablename__ == 'quota' + with dbcleanup(session, cls): + name, description = 'a', 'b' + obj = cls(name, description) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.name == name + assert stored_obj.description == description + assert stored_obj.bytes == 0 + assert stored_obj.operation == '=' + assert stored_obj.deleted is False + + +def test_Role(model, session): + cls = model.Role + assert cls.__tablename__ == 'role' + with dbcleanup(session, cls): + name, description = 'a', 'b' + obj = cls(name, description) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.update_time + assert stored_obj.name == name + assert stored_obj.description == description + assert stored_obj.type == model.Role.types.SYSTEM + assert stored_obj.deleted is False + + def test_UserAddress(model, session, user): cls = model.UserAddress assert cls.__tablename__ == 'user_address' @@ -217,6 +200,23 @@ def test_VisualizationRevision(model, session, visualization): assert stored_obj.config is None +def test_WorkerProcess(model, session): + cls = model.WorkerProcess + assert cls.__tablename__ == 'worker_process' + with dbcleanup(session, cls): + server_name, hostname = 'a', 'b' + obj = cls(server_name, hostname) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.server_name == server_name + assert stored_obj.hostname == hostname + assert stored_obj.pid is None + assert stored_obj.update_time + + @pytest.fixture(scope='module') def model(): db_uri = 'sqlite:///:memory:' From e4dc5083bde8cbe293685405462711c5c5be1722 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 3 Jun 2021 12:31:04 -0400 Subject: [PATCH 007/225] Import sqlalchemy.Unicode; do not refer via types Why: 1. Used in multiple Column definitions (moved from mapping.py) 2. `types` is used as a local variable in many places across the module; having the same identifier as an import creates unnecessary ambiguity. --- lib/galaxy/model/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3eab54810c4..13cc01b3faa 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -53,7 +53,7 @@ from sqlalchemy import ( true, tuple_, type_coerce, - types, + Unicode, UniqueConstraint, VARCHAR, ) @@ -3692,8 +3692,8 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot @type_id.expression # type: ignore def type_id(cls): - return ((type_coerce(cls.content_type, types.Unicode) + '-' - + type_coerce(cls.id, types.Unicode)).label('type_id')) + return ((type_coerce(cls.content_type, Unicode) + '-' + + type_coerce(cls.id, Unicode)).label('type_id')) class HistoryDatasetAssociationHistory(RepresentById): @@ -4634,8 +4634,8 @@ class HistoryDatasetCollectionAssociation(DatasetCollectionInstance, @type_id.expression # type: ignore def type_id(cls): - return ((type_coerce(cls.content_type, types.Unicode) + '-' - + type_coerce(cls.id, types.Unicode)).label('type_id')) + return ((type_coerce(cls.content_type, Unicode) + '-' + + type_coerce(cls.id, Unicode)).label('type_id')) @property def job_source_type(self): From 5690c39006cd4a0e9edf659a6870479aa3ee196f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 3 Jun 2021 13:22:04 -0400 Subject: [PATCH 008/225] Map UserAction declaratively Also: - Refactor constructor: - Do not pass id as arg - Pass reference to user object instead of user.id as arg - Do not pass create_time as arg NOTE: I did not add an 'actions' attribute to User; I assume we don't anticipate any need for that. Same for a UserAction.session attribute + relationship. --- lib/galaxy/model/__init__.py | 19 ++++++++++++++----- lib/galaxy/model/mapping.py | 18 +----------------- test/unit/model/test_model.py | 31 ++++++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 13cc01b3faa..df420490cee 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6798,11 +6798,20 @@ class UserPreference(RepresentById): self.value = value -class UserAction(RepresentById): - def __init__(self, id=None, create_time=None, user_id=None, session_id=None, action=None, params=None, context=None): - self.id = id - self.create_time = create_time - self.user_id = user_id +class UserAction(Base, RepresentById): + __tablename__ = 'user_action' + + id = Column("id", Integer, primary_key=True) + create_time = Column("create_time", DateTime, default=now) + user_id = Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True) + session_id = Column("session_id", Integer, ForeignKey("galaxy_session.id"), index=True) + action = Column("action", Unicode(255)) + context = Column("context", Unicode(512)) + params = Column("params", Unicode(1024)) + user = relationship('User') + + def __init__(self, user=None, session_id=None, action=None, params=None, context=None): + self.user = user self.session_id = session_id self.action = action self.params = params diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 705e6319104..beaa93c2c98 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1463,7 +1463,6 @@ model.LibraryDatasetCollectionRatingAssociation.table = Table( Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), Column("rating", Integer, index=True)) -# User tables. model.UserPreference.table = Table( "user_preference", metadata, Column("id", Integer, primary_key=True), @@ -1471,16 +1470,6 @@ model.UserPreference.table = Table( Column("name", Unicode(255), index=True), Column("value", Text)) -model.UserAction.table = Table( - "user_action", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("session_id", Integer, ForeignKey("galaxy_session.id"), index=True), - Column("action", Unicode(255)), - Column("context", Unicode(512)), - Column("params", Unicode(1024))) - model.APIKeys.table = Table( "api_keys", metadata, Column("id", Integer, primary_key=True), @@ -1554,14 +1543,9 @@ def simple_mapping(model, **kwds): mapper_registry.map_imperatively(model, model.table, properties=kwds) -# User tables. mapper_registry.map_imperatively(model.UserPreference, model.UserPreference.table, properties={}) -mapper_registry.map_imperatively(model.UserAction, model.UserAction.table, properties=dict( - # user=relation( model.User.mapper ) - user=relation(model.User) -)) -mapper_registry.map_imperatively(model.APIKeys, model.APIKeys.table, properties={}) +mapper_registry.map_imperatively(model.APIKeys, model.APIKeys.table, properties={}) mapper_registry.map_imperatively(model.FormValues, model.FormValues.table, properties=dict( form_definition=relation(model.FormDefinition, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 6cb7f3bcfd9..e67f9bd9b2d 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -152,6 +152,25 @@ def test_Role(model, session): assert stored_obj.deleted is False +def testUserAction(model, session, user, galaxy_session): + cls = model.UserAction + assert cls.__tablename__ == 'user_action' + with dbcleanup(session, cls): + action, params, context = 'a', 'b', 'c' + obj = cls(user, galaxy_session.id, action, params, context) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.create_time + assert stored_obj.user_id == user.id + assert stored_obj.session_id == galaxy_session.id + assert stored_obj.action == action + assert stored_obj.context == context + assert stored_obj.params == params + + def test_UserAddress(model, session, user): cls = model.UserAddress assert cls.__tablename__ == 'user_address' @@ -231,9 +250,9 @@ def session(model): @pytest.fixture -def user(model, session): - u = model.User(email='test@example.com', password='password') - yield from dbcleanup_wrapper(session, u) +def galaxy_session(model, session, user): + s = model.GalaxySession() + yield from dbcleanup_wrapper(session, s) @pytest.fixture @@ -243,6 +262,12 @@ def page(model, session, user): yield from dbcleanup_wrapper(session, p) +@pytest.fixture +def user(model, session): + u = model.User(email='test@example.com', password='password') + yield from dbcleanup_wrapper(session, u) + + @pytest.fixture def visualization(model, session, user): v = model.Visualization() From 4e0fdf76a7379ef7651a0343f203f0f849abdf1a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 3 Jun 2021 13:48:53 -0400 Subject: [PATCH 009/225] Restore accidentally deleted asserts (index, unique) --- test/unit/model/test_model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index e67f9bd9b2d..57dca7a3ecb 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -68,6 +68,7 @@ def test_PSAAssociation(model, session): def test_PSACode(model, session): cls = model.PSACode assert cls.__tablename__ == 'psa_code' + assert has_unique_constraint(cls.__table__, ('code', 'email')) with dbcleanup(session, cls): email, code = 'a', 'b' obj = cls(email, code) @@ -203,6 +204,7 @@ def test_UserAddress(model, session, user): def test_VisualizationRevision(model, session, visualization): cls = model.VisualizationRevision assert cls.__tablename__ == 'visualization_revision' + assert has_index(cls.__table__, ('dbkey',)) with dbcleanup(session, cls): obj = cls() obj.visualization = visualization @@ -222,6 +224,7 @@ def test_VisualizationRevision(model, session, visualization): def test_WorkerProcess(model, session): cls = model.WorkerProcess assert cls.__tablename__ == 'worker_process' + assert has_unique_constraint(cls.__table__, ('server_name', 'hostname')) with dbcleanup(session, cls): server_name, hostname = 'a', 'b' obj = cls(server_name, hostname) From 0dae0d9340431cff19c13bf01f96b07a6850f72f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 3 Jun 2021 15:47:59 -0400 Subject: [PATCH 010/225] Map UserAuthnzToken declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 20 ++------------------ test/unit/model/test_model.py | 21 +++++++++++++++++++++ 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index df420490cee..3dd021752f4 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6310,9 +6310,19 @@ class PSAPartial(Base, PartialMixin, RepresentById): cls.sa_session.delete(partial) -class UserAuthnzToken(UserMixin, RepresentById): +class UserAuthnzToken(Base, UserMixin, RepresentById): + __tablename__ = 'oidc_user_authnz_tokens' __table_args__ = (UniqueConstraint('provider', 'uid'),) + id = Column('id', Integer, primary_key=True) + user_id = Column('user_id', Integer, ForeignKey("galaxy_user.id"), index=True) + uid = Column('uid', VARCHAR(255)) + provider = Column('provider', VARCHAR(32)) + extra_data = Column('extra_data', MutableJSONType, nullable=True) + lifetime = Column('lifetime', Integer) + assoc_type = Column('assoc_type', VARCHAR(64)) + user = relationship('User', back_populates='social_auth') + # This static property is set at: galaxy.authnz.psa_authnz.PSAAuthnz sa_session = None diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index beaa93c2c98..0d2c9a6f378 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -29,7 +29,6 @@ from sqlalchemy import ( true, Unicode, UniqueConstraint, - VARCHAR ) from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.orderinglist import ordering_list @@ -76,16 +75,6 @@ model.User.table = Table( Column("active", Boolean, index=True, default=True, nullable=False), Column("activation_token", TrimmedString(64), nullable=True, index=True)) -model.UserAuthnzToken.table = Table( - "oidc_user_authnz_tokens", metadata, - Column('id', Integer, primary_key=True), - Column('user_id', Integer, ForeignKey("galaxy_user.id"), index=True), - Column('uid', VARCHAR(255)), - Column('provider', VARCHAR(32)), - Column('extra_data', MutableJSONType, nullable=True), - Column('lifetime', Integer), - Column('assoc_type', VARCHAR(64))) - model.CustosAuthnzToken.table = Table( "custos_authnz_token", metadata, Column('id', Integer, primary_key=True), @@ -1564,12 +1553,6 @@ mapper_registry.map_imperatively(model.FormDefinitionCurrent, model.FormDefiniti primaryjoin=(model.FormDefinitionCurrent.table.c.latest_form_id == model.FormDefinition.table.c.id)) )) -mapper_registry.map_imperatively(model.UserAuthnzToken, model.UserAuthnzToken.table, properties=dict( - user=relation(model.User, - primaryjoin=(model.UserAuthnzToken.table.c.user_id == model.User.table.c.id), - backref='social_auth') -)) - mapper_registry.map_imperatively(model.CustosAuthnzToken, model.CustosAuthnzToken.table, properties=dict( user=relation(model.User, primaryjoin=(model.CustosAuthnzToken.table.c.user_id == model.User.table.c.id), @@ -1581,7 +1564,7 @@ mapper_registry.map_imperatively(model.CloudAuthz, model.CloudAuthz.table, prope primaryjoin=(model.CloudAuthz.table.c.user_id == model.User.table.c.id), backref='cloudauthz'), authn=relation(model.UserAuthnzToken, - primaryjoin=(model.CloudAuthz.table.c.authn_id == model.UserAuthnzToken.table.c.id), + primaryjoin=(model.CloudAuthz.table.c.authn_id == model.UserAuthnzToken.id), backref='cloudauthz') )) @@ -1773,6 +1756,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( galaxy_sessions=relation(model.GalaxySession, backref="user", order_by=desc(model.GalaxySession.table.c.update_time)), + social_auth=relation(model.UserAuthnzToken, back_populates='user'), stored_workflow_menu_entries=relation(model.StoredWorkflowMenuEntry, primaryjoin=( (model.StoredWorkflowMenuEntry.table.c.user_id == model.User.table.c.id) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 57dca7a3ecb..43f61812c6e 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -201,6 +201,27 @@ def test_UserAddress(model, session, user): assert stored_obj.user == user +def test_UserAuthnzToken(model, session, user): + cls = model.UserAuthnzToken + assert cls.__tablename__ == 'oidc_user_authnz_tokens' + assert has_unique_constraint(cls.__table__, ('provider', 'uid')) + with dbcleanup(session, cls): + provider, uid, extra_data, lifetime, assoc_type = 'a', 'b', 'c', 1, 'd' + obj = cls(provider, uid, extra_data, lifetime, assoc_type, user) + persist(session, obj) + + stmt = select(cls) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id + assert stored_obj.id + assert stored_obj.user_id == user.id + assert stored_obj.uid == uid + assert stored_obj.provider == provider + assert stored_obj.extra_data == extra_data + assert stored_obj.lifetime == lifetime + assert stored_obj.assoc_type == assoc_type + + def test_VisualizationRevision(model, session, visualization): cls = model.VisualizationRevision assert cls.__tablename__ == 'visualization_revision' From 0af13a3b2babb655fdb39d1994a97ee3bc0a63d9 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 3 Jun 2021 20:20:21 -0400 Subject: [PATCH 011/225] Map CloudAuthz declaratively, refactor tests Because there may be more than one instance of the same model in one test: 1. Generate random values for fields with unique constraints. 2. Query for stored object by object id to ensure we get the right object. --- lib/galaxy/model/__init__.py | 23 +++++- lib/galaxy/model/mapping.py | 23 +----- test/unit/model/test_model.py | 141 +++++++++++++++++++++------------- 3 files changed, 107 insertions(+), 80 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3dd021752f4..6ac4309dd7a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2343,6 +2343,7 @@ class GroupRoleAssociation(RepresentById): class Role(Base, Dictifiable, RepresentById): __tablename__ = 'role' + id = Column('id', Integer, primary_key=True) create_time = Column('create_time', DateTime, default=now) update_time = Column('update_time', DateTime, default=now, onupdate=now) @@ -6321,6 +6322,7 @@ class UserAuthnzToken(Base, UserMixin, RepresentById): extra_data = Column('extra_data', MutableJSONType, nullable=True) lifetime = Column('lifetime', Integer) assoc_type = Column('assoc_type', VARCHAR(64)) + cloudauthz = relationship('CloudAuthz', back_populates='authn') user = relationship('User', back_populates='social_auth') # This static property is set at: galaxy.authnz.psa_authnz.PSAAuthnz @@ -6441,14 +6443,27 @@ class CustosAuthnzToken(RepresentById): self.refresh_expiration_time = refresh_expiration_time -class CloudAuthz(_HasTable): - def __init__(self, user_id, provider, config, authn_id, description=""): - self.id = None +class CloudAuthz(Base, _HasTable): + __tablename__ = 'cloudauthz' + + id = Column('id', Integer, primary_key=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + provider = Column('provider', String(255)) + config = Column('config', MutableJSONType) + authn_id = Column('authn_id', Integer, ForeignKey('oidc_user_authnz_tokens.id'), index=True) + tokens = Column('tokens', MutableJSONType) + last_update = Column('last_update', DateTime) + last_activity = Column('last_activity', DateTime) + description = Column('description', TEXT) + create_time = Column('create_time', DateTime, default=now) + user = relationship('User', back_populates='cloudauthz') + authn = relationship('UserAuthnzToken', back_populates='cloudauthz') + + def __init__(self, user_id, provider, config, authn_id, description=None): self.user_id = user_id self.provider = provider self.config = config self.authn_id = authn_id - self.tokens = None self.last_update = datetime.now() self.last_activity = datetime.now() self.description = description diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 0d2c9a6f378..7df10369f0e 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -90,19 +90,6 @@ model.CustosAuthnzToken.table = Table( UniqueConstraint("external_user_id", "provider"), ) -model.CloudAuthz.table = Table( - "cloudauthz", metadata, - Column('id', Integer, primary_key=True), - Column('user_id', Integer, ForeignKey("galaxy_user.id"), index=True), - Column('provider', String(255)), - Column('config', MutableJSONType), - Column('authn_id', Integer, ForeignKey("oidc_user_authnz_tokens.id"), index=True), - Column('tokens', MutableJSONType), - Column('last_update', DateTime), - Column('last_activity', DateTime), - Column('description', TEXT), - Column('create_time', DateTime, default=now)) - model.PasswordResetToken.table = Table( "password_reset_token", metadata, Column("token", String(32), primary_key=True, unique=True, index=True), @@ -1559,15 +1546,6 @@ mapper_registry.map_imperatively(model.CustosAuthnzToken, model.CustosAuthnzToke backref='custos_auth') )) -mapper_registry.map_imperatively(model.CloudAuthz, model.CloudAuthz.table, properties=dict( - user=relation(model.User, - primaryjoin=(model.CloudAuthz.table.c.user_id == model.User.table.c.id), - backref='cloudauthz'), - authn=relation(model.UserAuthnzToken, - primaryjoin=(model.CloudAuthz.table.c.authn_id == model.UserAuthnzToken.id), - backref='cloudauthz') -)) - simple_mapping(model.DynamicTool) simple_mapping(model.HistoryDatasetAssociation, @@ -1743,6 +1721,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( addresses=relation(model.UserAddress, back_populates='user', order_by=desc(model.UserAddress.update_time)), + cloudauthz=relation(model.CloudAuthz, back_populates='user'), histories=relation(model.History, backref="user", order_by=desc(model.History.update_time)), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 43f61812c6e..a7e7f3bcd06 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1,3 +1,4 @@ +import random from contextlib import contextmanager import pytest @@ -10,21 +11,30 @@ from sqlalchemy import ( import galaxy.model.mapping as mapping -def test_Group(model, session): - cls = model.Group - assert cls.__tablename__ == 'galaxy_group' +def test_CloudAuthz(model, session, user, user_authnz_token): + cls = model.CloudAuthz + assert cls.__tablename__ == 'cloudauthz' with dbcleanup(session, cls): - name = 'a' - obj = cls(name) - persist(session, obj) + provider, config, description = 'a', 'b', 'c' + obj = cls(user.id, provider, config, user_authnz_token.id, description) + obj.user = user + obj.authn = user_authnz_token + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id + assert stored_obj.user_id == user.id + assert stored_obj.provider == provider + assert stored_obj.config == config + assert stored_obj.authn_id == user_authnz_token.id + assert stored_obj.description == description + assert stored_obj.tokens is None + assert stored_obj.last_update + assert stored_obj.last_activity assert stored_obj.create_time - assert stored_obj.update_time - assert stored_obj.name == name - assert stored_obj.deleted is False + assert stored_obj.user == user + assert stored_obj.authn == user_authnz_token def test_PageRevision(model, session, page): @@ -33,11 +43,11 @@ def test_PageRevision(model, session, page): with dbcleanup(session, cls): obj = cls() obj.page = page - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time assert stored_obj.page_id @@ -52,11 +62,11 @@ def test_PSAAssociation(model, session): with dbcleanup(session, cls): server_url, handle, secret, issued, lifetime, assoc_type = 'a', 'b', 'c', 1, 2, 'd' obj = cls(server_url, handle, secret, issued, lifetime, assoc_type) - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.server_url == server_url assert stored_obj.handle == handle assert stored_obj.secret == secret @@ -70,13 +80,13 @@ def test_PSACode(model, session): assert cls.__tablename__ == 'psa_code' assert has_unique_constraint(cls.__table__, ('code', 'email')) with dbcleanup(session, cls): - email, code = 'a', 'b' + email, code = 'a', get_random_string() obj = cls(email, code) - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.email == email assert stored_obj.code == code @@ -87,11 +97,11 @@ def test_PSANonce(model, session): with dbcleanup(session, cls): server_url, timestamp, salt = 'a', 1, 'b' obj = cls(server_url, timestamp, salt) - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.server_url assert stored_obj.timestamp == timestamp assert stored_obj.salt == salt @@ -103,11 +113,11 @@ def test_PSAPartial(model, session): with dbcleanup(session, cls): token, data, next_step, backend = 'a', 'b', 1, 'c' obj = cls(token, data, next_step, backend) - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.token == token assert stored_obj.data == data assert stored_obj.next_step == next_step @@ -118,13 +128,13 @@ def test_Quota(model, session): cls = model.Quota assert cls.__tablename__ == 'quota' with dbcleanup(session, cls): - name, description = 'a', 'b' + name, description = get_random_string(), 'b' obj = cls(name, description) - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time assert stored_obj.name == name @@ -138,13 +148,13 @@ def test_Role(model, session): cls = model.Role assert cls.__tablename__ == 'role' with dbcleanup(session, cls): - name, description = 'a', 'b' + name, description = get_random_string(), 'b' obj = cls(name, description) - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time assert stored_obj.name == name @@ -159,17 +169,18 @@ def testUserAction(model, session, user, galaxy_session): with dbcleanup(session, cls): action, params, context = 'a', 'b', 'c' obj = cls(user, galaxy_session.id, action, params, context) - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.user_id == user.id assert stored_obj.session_id == galaxy_session.id assert stored_obj.action == action assert stored_obj.context == context assert stored_obj.params == params + assert stored_obj.user == user def test_UserAddress(model, session, user): @@ -179,11 +190,11 @@ def test_UserAddress(model, session, user): desc, name, institution, address, city, state, postal_code, country, phone = \ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i' obj = cls(user, desc, name, institution, address, city, state, postal_code, country, phone) - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time assert stored_obj.user_id == user.id @@ -201,25 +212,28 @@ def test_UserAddress(model, session, user): assert stored_obj.user == user -def test_UserAuthnzToken(model, session, user): +def test_UserAuthnzToken(model, session, user, cloud_authz): cls = model.UserAuthnzToken assert cls.__tablename__ == 'oidc_user_authnz_tokens' assert has_unique_constraint(cls.__table__, ('provider', 'uid')) with dbcleanup(session, cls): - provider, uid, extra_data, lifetime, assoc_type = 'a', 'b', 'c', 1, 'd' + provider, uid, extra_data, lifetime, assoc_type = get_random_string(), 'b', 'c', 1, 'd' obj = cls(provider, uid, extra_data, lifetime, assoc_type, user) - persist(session, obj) + obj.cloudauthz.append(cloud_authz) + obj.user = user + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.user_id == user.id assert stored_obj.uid == uid assert stored_obj.provider == provider assert stored_obj.extra_data == extra_data assert stored_obj.lifetime == lifetime assert stored_obj.assoc_type == assoc_type + assert stored_obj.user == user + assert cloud_authz in stored_obj.cloudauthz def test_VisualizationRevision(model, session, visualization): @@ -229,11 +243,11 @@ def test_VisualizationRevision(model, session, visualization): with dbcleanup(session, cls): obj = cls() obj.visualization = visualization - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time assert stored_obj.visualization_id @@ -247,13 +261,13 @@ def test_WorkerProcess(model, session): assert cls.__tablename__ == 'worker_process' assert has_unique_constraint(cls.__table__, ('server_name', 'hostname')) with dbcleanup(session, cls): - server_name, hostname = 'a', 'b' + server_name, hostname = get_random_string(), 'a' obj = cls(server_name, hostname) - persist(session, obj) + obj_id = persist(session, obj) - stmt = select(cls) + stmt = select(cls).filter(cls.id == obj_id) stored_obj = session.execute(stmt).scalar_one() - assert stored_obj.id + assert stored_obj.id == obj_id assert stored_obj.server_name == server_name assert stored_obj.hostname == hostname assert stored_obj.pid is None @@ -273,6 +287,12 @@ def session(model): Session.remove() # Ensures we get a new session for each test +@pytest.fixture +def cloud_authz(model, session, user, user_authnz_token): + ca = model.CloudAuthz(user.id, 'a', 'b', user_authnz_token.id, 'c') + yield from dbcleanup_wrapper(session, ca) + + @pytest.fixture def galaxy_session(model, session, user): s = model.GalaxySession() @@ -292,6 +312,12 @@ def user(model, session): yield from dbcleanup_wrapper(session, u) +@pytest.fixture +def user_authnz_token(model, session, user): + t = model.UserAuthnzToken('a', 'b', 'c', 1, 'd', user) + yield from dbcleanup_wrapper(session, t) + + @pytest.fixture def visualization(model, session, user): v = model.Visualization() @@ -318,8 +344,10 @@ def dbcleanup_wrapper(session, obj): def persist(session, obj): session.add(obj) session.flush() + obj_id = obj.id # Remove from session, so that on a subsequent load we get a *new* obj from the db session.expunge(obj) + return obj_id def has_unique_constraint(table, fields): @@ -335,3 +363,8 @@ def has_index(table, fields): col_names = {c.name for c in index.columns} if set(fields) == col_names: return True + + +def get_random_string(): + """Generate unique values to accommodate unique constraints.""" + return str(random.random()) From 9c8080f878f18afbf3ad412d3825fbf84c47d55f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 3 Jun 2021 23:06:05 -0400 Subject: [PATCH 012/225] Map CustosAuthnzToken declaratively --- lib/galaxy/model/__init__.py | 21 ++++++++++++++++++--- lib/galaxy/model/mapping.py | 22 +--------------------- test/unit/model/test_model.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6ac4309dd7a..244b604e97c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -49,6 +49,7 @@ from sqlalchemy import ( select, String, TEXT, + Text, text, true, tuple_, @@ -6428,11 +6429,25 @@ class UserAuthnzToken(Base, UserMixin, RepresentById): return instance -class CustosAuthnzToken(RepresentById): +class CustosAuthnzToken(Base, RepresentById): + __tablename__ = 'custos_authnz_token' + __table_args__ = ( + UniqueConstraint('user_id', 'external_user_id', 'provider'), + UniqueConstraint('external_user_id', 'provider'), + ) + + id = Column('id', Integer, primary_key=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id')) + external_user_id = Column('external_user_id', String(64)) + provider = Column('provider', String(255)) + access_token = Column('access_token', Text) + id_token = Column('id_token', Text) + refresh_token = Column('refresh_token', Text) + expiration_time = Column('expiration_time', DateTime) + refresh_expiration_time = Column('refresh_expiration_time', DateTime) + user = relationship('User', back_populates='custos_auth') def __init__(self, user, external_user_id, provider, access_token, id_token, refresh_token, expiration_time, refresh_expiration_time): - self.id = None - # 'user' is a backref to model.User self.user = user self.external_user_id = external_user_id self.provider = provider diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 7df10369f0e..8a8562d61e5 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -75,21 +75,6 @@ model.User.table = Table( Column("active", Boolean, index=True, default=True, nullable=False), Column("activation_token", TrimmedString(64), nullable=True, index=True)) -model.CustosAuthnzToken.table = Table( - "custos_authnz_token", metadata, - Column('id', Integer, primary_key=True), - Column('user_id', Integer, ForeignKey("galaxy_user.id")), - Column('external_user_id', String(64)), - Column('provider', String(255)), - Column('access_token', Text), - Column('id_token', Text), - Column('refresh_token', Text), - Column("expiration_time", DateTime), - Column("refresh_expiration_time", DateTime), - UniqueConstraint("user_id", "external_user_id", "provider"), - UniqueConstraint("external_user_id", "provider"), -) - model.PasswordResetToken.table = Table( "password_reset_token", metadata, Column("token", String(32), primary_key=True, unique=True, index=True), @@ -1540,12 +1525,6 @@ mapper_registry.map_imperatively(model.FormDefinitionCurrent, model.FormDefiniti primaryjoin=(model.FormDefinitionCurrent.table.c.latest_form_id == model.FormDefinition.table.c.id)) )) -mapper_registry.map_imperatively(model.CustosAuthnzToken, model.CustosAuthnzToken.table, properties=dict( - user=relation(model.User, - primaryjoin=(model.CustosAuthnzToken.table.c.user_id == model.User.table.c.id), - backref='custos_auth') -)) - simple_mapping(model.DynamicTool) simple_mapping(model.HistoryDatasetAssociation, @@ -1722,6 +1701,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( back_populates='user', order_by=desc(model.UserAddress.update_time)), cloudauthz=relation(model.CloudAuthz, back_populates='user'), + custos_auth=relation(model.CustosAuthnzToken, back_populates='user'), histories=relation(model.History, backref="user", order_by=desc(model.History.update_time)), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index a7e7f3bcd06..bbcaa5acbed 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1,5 +1,6 @@ import random from contextlib import contextmanager +from datetime import datetime import pytest from sqlalchemy import ( @@ -37,6 +38,36 @@ def test_CloudAuthz(model, session, user, user_authnz_token): assert stored_obj.authn == user_authnz_token +def test_CustosAuthnzToken(model, session, user): + cls = model.CustosAuthnzToken + assert cls.__tablename__ == 'custos_authnz_token' + assert has_unique_constraint(cls.__table__, ('user_id', 'external_user_id', 'provider')) + assert has_unique_constraint(cls.__table__, ('external_user_id', 'provider')) + with dbcleanup(session, cls): + external_user_id = get_random_string() + provider = get_random_string() + access_token = 'c' + id_token = 'd' + refresh_token = 'e' + expiration_time = datetime.now() + refresh_expiration_time = datetime.now() + obj = cls(user, external_user_id, provider, access_token, id_token, refresh_token, + expiration_time, refresh_expiration_time) + obj_id = persist(session, obj) + + stmt = select(cls).filter(cls.id == obj_id) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id == obj_id + assert stored_obj.external_user_id == external_user_id + assert stored_obj.provider == provider + assert stored_obj.access_token == access_token + assert stored_obj.id_token == id_token + assert stored_obj.refresh_token == refresh_token + assert stored_obj.expiration_time == expiration_time + assert stored_obj.refresh_expiration_time == refresh_expiration_time + assert stored_obj.user == user + + def test_PageRevision(model, session, page): cls = model.PageRevision assert cls.__tablename__ == 'page_revision' From 1e97aecfa644398f097d3249a771b031586a55a5 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 9 Jun 2021 15:58:27 -0400 Subject: [PATCH 013/225] Persist model fixtures before using them in tests Because if we don't, those objects do NOT have an id, so testing in a case like this is incorrect: with foo-fixture: b = Bar(foo_id=foo-fixture.id) persist(session, b) # then load b from database assert b.foo_id == foo-fixture.id # this passes, but it's comparing None to None! If we persist, then foo-fixture.id and foo.id contain actual values. --- test/unit/model/test_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index bbcaa5acbed..b54b26d25ff 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -368,6 +368,7 @@ def dbcleanup(session, cls): def dbcleanup_wrapper(session, obj): + persist(session, obj) with dbcleanup(session, type(obj)): yield obj From dedf466f4929633094e2c4daae7a265e57bb2591 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 9 Jun 2021 16:22:47 -0400 Subject: [PATCH 014/225] Test mapped relationships explicitly --- test/unit/model/test_model.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index b54b26d25ff..60ee0a9676a 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -18,8 +18,6 @@ def test_CloudAuthz(model, session, user, user_authnz_token): with dbcleanup(session, cls): provider, config, description = 'a', 'b', 'c' obj = cls(user.id, provider, config, user_authnz_token.id, description) - obj.user = user - obj.authn = user_authnz_token obj_id = persist(session, obj) stmt = select(cls).filter(cls.id == obj_id) @@ -34,8 +32,8 @@ def test_CloudAuthz(model, session, user, user_authnz_token): assert stored_obj.last_update assert stored_obj.last_activity assert stored_obj.create_time - assert stored_obj.user == user - assert stored_obj.authn == user_authnz_token + assert stored_obj.user.id == user.id + assert stored_obj.authn.id == user_authnz_token.id def test_CustosAuthnzToken(model, session, user): @@ -81,10 +79,11 @@ def test_PageRevision(model, session, page): assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time - assert stored_obj.page_id + assert stored_obj.page_id == page.id assert stored_obj.title is None assert stored_obj.content is None assert stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT + assert stored_obj.page.id == page.id def test_PSAAssociation(model, session): @@ -211,7 +210,7 @@ def testUserAction(model, session, user, galaxy_session): assert stored_obj.action == action assert stored_obj.context == context assert stored_obj.params == params - assert stored_obj.user == user + assert stored_obj.user.id == user.id def test_UserAddress(model, session, user): @@ -251,7 +250,6 @@ def test_UserAuthnzToken(model, session, user, cloud_authz): provider, uid, extra_data, lifetime, assoc_type = get_random_string(), 'b', 'c', 1, 'd' obj = cls(provider, uid, extra_data, lifetime, assoc_type, user) obj.cloudauthz.append(cloud_authz) - obj.user = user obj_id = persist(session, obj) stmt = select(cls).filter(cls.id == obj_id) @@ -263,7 +261,7 @@ def test_UserAuthnzToken(model, session, user, cloud_authz): assert stored_obj.extra_data == extra_data assert stored_obj.lifetime == lifetime assert stored_obj.assoc_type == assoc_type - assert stored_obj.user == user + assert stored_obj.user.id == user.id assert cloud_authz in stored_obj.cloudauthz @@ -272,8 +270,8 @@ def test_VisualizationRevision(model, session, visualization): assert cls.__tablename__ == 'visualization_revision' assert has_index(cls.__table__, ('dbkey',)) with dbcleanup(session, cls): - obj = cls() - obj.visualization = visualization + visualization, title, dbkey, config = visualization, 'a', 'b', 'c' + obj = cls(visualization, title, dbkey, config) obj_id = persist(session, obj) stmt = select(cls).filter(cls.id == obj_id) @@ -281,10 +279,11 @@ def test_VisualizationRevision(model, session, visualization): assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time - assert stored_obj.visualization_id - assert stored_obj.title is None - assert stored_obj.dbkey is None - assert stored_obj.config is None + assert stored_obj.visualization_id == visualization.id + assert stored_obj.title == title + assert stored_obj.dbkey == dbkey + assert stored_obj.config == config + assert stored_obj.visualization.id == visualization.id def test_WorkerProcess(model, session): From 272fcefe81479bbd1ccfff510c7ba0f438357aa3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 9 Jun 2021 16:29:58 -0400 Subject: [PATCH 015/225] MapAPIKeys declaratively --- lib/galaxy/managers/users.py | 2 +- lib/galaxy/model/__init__.py | 13 ++++++++++--- lib/galaxy/model/mapping.py | 13 ++----------- test/unit/model/test_model.py | 16 ++++++++++++++++ 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py index 3407a6bd756..52dabfe1c70 100644 --- a/lib/galaxy/managers/users.py +++ b/lib/galaxy/managers/users.py @@ -237,7 +237,7 @@ class UserManager(base.ModelManager, deletable.PurgableManagerMixin): return schema.BootstrapAdminUser() sa_session = sa_session or self.app.model.session try: - provided_key = sa_session.query(self.app.model.APIKeys).filter(self.app.model.APIKeys.table.c.key == api_key).one() + provided_key = sa_session.query(self.app.model.APIKeys).filter(self.app.model.APIKeys.key == api_key).one() except NoResultFound: raise exceptions.AuthenticationFailed('Provided API key is not valid.') if provided_key.user.deleted: diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 244b604e97c..e7462c57d77 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6858,9 +6858,16 @@ class UserAction(Base, RepresentById): self.context = context -class APIKeys(RepresentById): - def __init__(self, id=None, user_id=None, key=None): - self.id = id +class APIKeys(Base, RepresentById): + __tablename__ = 'api_keys' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + key = Column('key', TrimmedString(32), index=True, unique=True) + user = relationship('User', back_populates='api_keys') + + def __init__(self, user_id=None, key=None): self.user_id = user_id self.key = key diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 8a8562d61e5..56994ee592a 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1431,13 +1431,6 @@ model.UserPreference.table = Table( Column("name", Unicode(255), index=True), Column("value", Text)) -model.APIKeys.table = Table( - "api_keys", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("key", TrimmedString(32), index=True, unique=True)) - CleanupEvent_table = Table("cleanup_event", metadata, Column("id", Integer, primary_key=True), Column("create_time", DateTime, default=now), @@ -1506,8 +1499,6 @@ def simple_mapping(model, **kwds): mapper_registry.map_imperatively(model.UserPreference, model.UserPreference.table, properties={}) -mapper_registry.map_imperatively(model.APIKeys, model.APIKeys.table, properties={}) - mapper_registry.map_imperatively(model.FormValues, model.FormValues.table, properties=dict( form_definition=relation(model.FormDefinition, primaryjoin=(model.FormValues.table.c.form_definition_id == model.FormDefinition.table.c.id)) @@ -1731,8 +1722,8 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( values=relation(model.FormValues, primaryjoin=(model.User.table.c.form_values_id == model.FormValues.table.c.id)), api_keys=relation(model.APIKeys, - backref="user", - order_by=desc(model.APIKeys.table.c.create_time)), + back_populates="user", + order_by=desc(model.APIKeys.create_time)), )) mapper_registry.map_imperatively(model.PasswordResetToken, model.PasswordResetToken.table, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 60ee0a9676a..1fea214a0f2 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -12,6 +12,22 @@ from sqlalchemy import ( import galaxy.model.mapping as mapping +def testAPIKeys(model, session, user): + cls = model.APIKeys + assert cls.__tablename__ == 'api_keys' + with dbcleanup(session, cls): + user_id, key = user.id, 'a' + obj = cls(user_id, key) + obj_id = persist(session, obj) + + stmt = select(cls).filter(cls.id == obj_id) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id == obj_id + assert stored_obj.user_id == user_id + assert stored_obj.key == key + assert stored_obj.user.id == user.id + + def test_CloudAuthz(model, session, user, user_authnz_token): cls = model.CloudAuthz assert cls.__tablename__ == 'cloudauthz' From 0bc59ab032c4aaf2fca35b16c84c18074beccfe8 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 10 Jun 2021 23:53:32 -0400 Subject: [PATCH 016/225] Map DynamicTool declaratively --- lib/galaxy/model/__init__.py | 19 ++++++++++++++- lib/galaxy/model/mapping.py | 24 ++---------------- test/unit/model/test_model.py | 46 +++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e7462c57d77..7475fcb9d04 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -80,6 +80,7 @@ import galaxy.util from galaxy.model.custom_types import ( MutableJSONType, TrimmedString, + UUIDType, ) from galaxy.model.item_attrs import get_item_annotation_str, UsesAnnotations from galaxy.model.orm.now import now @@ -691,7 +692,23 @@ class PasswordResetToken(_HasTable): self.expiration_time = galaxy.model.orm.now.now() + timedelta(hours=24) -class DynamicTool(Dictifiable, RepresentById): +class DynamicTool(Base, Dictifiable, RepresentById): + __tablename__ = 'dynamic_tool' + + id = Column('id', Integer, primary_key=True) + uuid = Column('uuid', UUIDType()) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, index=True, default=now, onupdate=now) + tool_id = Column('tool_id', Unicode(255)) + tool_version = Column('tool_version', Unicode(255)) + tool_format = Column('tool_format', Unicode(255)) + tool_path = Column('tool_path', Unicode(255)) + tool_directory = Column('tool_directory', Unicode(255)) + hidden = Column('hidden', Boolean, default=True) + active = Column('active', Boolean, default=True) + value = Column('value', MutableJSONType) + workflow_steps = relationship('WorkflowStep', back_populates='dynamic_tool') + dict_collection_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden') dict_element_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden') diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 56994ee592a..ba8a51bdaf5 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -81,24 +81,6 @@ model.PasswordResetToken.table = Table( Column("expiration_time", DateTime), Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True)) - -model.DynamicTool.table = Table( - "dynamic_tool", metadata, - Column("id", Integer, primary_key=True), - Column("uuid", UUIDType()), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, index=True, default=now, onupdate=now), - Column("tool_id", Unicode(255)), - Column("tool_version", Unicode(255)), - Column("tool_format", Unicode(255)), - Column("tool_path", Unicode(255)), - Column("tool_directory", Unicode(255)), - Column("hidden", Boolean, default=True), - Column("active", Boolean, default=True), - Column("value", MutableJSONType), -) - - model.History.table = Table( "history", metadata, Column("id", Integer, primary_key=True), @@ -1516,8 +1498,6 @@ mapper_registry.map_imperatively(model.FormDefinitionCurrent, model.FormDefiniti primaryjoin=(model.FormDefinitionCurrent.table.c.latest_form_id == model.FormDefinition.table.c.id)) )) -simple_mapping(model.DynamicTool) - simple_mapping(model.HistoryDatasetAssociation, dataset=relation(model.Dataset, primaryjoin=(model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id), @@ -2188,8 +2168,8 @@ mapper_registry.map_imperatively(model.WorkflowStep, model.WorkflowStep.table, p primaryjoin=(model.Workflow.table.c.id == model.WorkflowStep.table.c.subworkflow_id), backref="parent_workflow_steps"), dynamic_tool=relation(model.DynamicTool, - primaryjoin=(model.DynamicTool.table.c.id == model.WorkflowStep.table.c.dynamic_tool_id), - backref="workflow_steps"), + primaryjoin=(model.DynamicTool.id == model.WorkflowStep.table.c.dynamic_tool_id), + back_populates="workflow_steps"), tags=relation(model.WorkflowStepTagAssociation, order_by=model.WorkflowStepTagAssociation.table.c.id, backref="workflow_steps"), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 1fea214a0f2..969938a6817 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -82,6 +82,39 @@ def test_CustosAuthnzToken(model, session, user): assert stored_obj.user == user +def test_DynamicTool(model, session, workflow_step): + cls = model.DynamicTool + assert cls.__tablename__ == 'dynamic_tool' + with dbcleanup(session, cls): + tool_format = 'a' + tool_id = 'b' + tool_version = 'c' + tool_path = 'd' + tool_directory = 'e' + uuid = None + active = True + hidden = True + value = 'f' + obj = cls(tool_format, tool_id, tool_version, tool_path, tool_directory, uuid, + active, hidden, value) + obj.workflow_steps.append(workflow_step) + obj_id = persist(session, obj) + + stmt = select(cls).filter(cls.id == obj_id) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id == obj_id + assert stored_obj.tool_format == tool_format + assert stored_obj.tool_id == tool_id + assert stored_obj.tool_version == tool_version + assert stored_obj.tool_path == tool_path + assert stored_obj.tool_directory == tool_directory + assert stored_obj.uuid + assert stored_obj.active == active + assert stored_obj.hidden == hidden + assert stored_obj.value == value + assert workflow_step in stored_obj.workflow_steps + + def test_PageRevision(model, session, page): cls = model.PageRevision assert cls.__tablename__ == 'page_revision' @@ -371,6 +404,19 @@ def visualization(model, session, user): yield from dbcleanup_wrapper(session, v) +@pytest.fixture +def workflow(model, session): + w = model.Workflow() + yield from dbcleanup_wrapper(session, w) + + +@pytest.fixture +def workflow_step(model, session, workflow): + s = model.WorkflowStep() + s.workflow = workflow + yield from dbcleanup_wrapper(session, s) + + @contextmanager def dbcleanup(session, cls): """ From c3b769269bd4e0332fe4a72ffdca52da10ce6d98 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 11 Jun 2021 00:32:22 -0400 Subject: [PATCH 017/225] Map DefaultUserPermission declaratively --- lib/galaxy/model/__init__.py | 10 +++++++++- lib/galaxy/model/mapping.py | 13 +------------ test/unit/model/test_model.py | 22 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7475fcb9d04..1d407a4168e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2512,7 +2512,15 @@ class LibraryDatasetDatasetAssociationPermissions(RepresentById): self.role = role -class DefaultUserPermissions(RepresentById): +class DefaultUserPermissions(Base, RepresentById): + __tablename__ = 'default_user_permissions' + id = Column('id', Integer, primary_key=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + action = Column('action', TEXT) + role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + user = relationship('User', back_populates='default_permissions') + role = relationship('Role') + def __init__(self, user, action, role): self.user = user self.action = action diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index ba8a51bdaf5..f680524cbca 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -327,13 +327,6 @@ model.LibraryDatasetDatasetAssociationPermissions.table = Table( nullable=True, index=True), Column("role_id", Integer, ForeignKey("role.id"), index=True)) -model.DefaultUserPermissions.table = Table( - "default_user_permissions", metadata, - Column("id", Integer, primary_key=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("action", TEXT), - Column("role_id", Integer, ForeignKey("role.id"), index=True)) - model.DefaultHistoryPermissions.table = Table( "default_history_permissions", metadata, Column("id", Integer, primary_key=True), @@ -1673,6 +1666,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( order_by=desc(model.UserAddress.update_time)), cloudauthz=relation(model.CloudAuthz, back_populates='user'), custos_auth=relation(model.CustosAuthnzToken, back_populates='user'), + default_permissions=relation(model.DefaultUserPermissions, back_populates='user'), histories=relation(model.History, backref="user", order_by=desc(model.History.update_time)), @@ -1719,11 +1713,6 @@ mapper_registry.map_imperatively(model.UserGroupAssociation, model.UserGroupAsso group=relation(model.Group, backref="users") )) -mapper_registry.map_imperatively(model.DefaultUserPermissions, model.DefaultUserPermissions.table, properties=dict( - user=relation(model.User, backref="default_permissions"), - role=relation(model.Role) -)) - mapper_registry.map_imperatively(model.DefaultHistoryPermissions, model.DefaultHistoryPermissions.table, properties=dict( history=relation(model.History, backref="default_permissions"), role=relation(model.Role) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 969938a6817..920a9927443 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -82,6 +82,22 @@ def test_CustosAuthnzToken(model, session, user): assert stored_obj.user == user +def test_DefaultUserPermissions(model, session, user, role): + cls = model.DefaultUserPermissions + assert cls.__tablename__ == 'default_user_permissions' + with dbcleanup(session, cls): + action = 'a' + obj = cls(user, action, role) + obj_id = persist(session, obj) + + stmt = select(cls).filter(cls.id == obj_id) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.action == action + assert stored_obj.role == role + + def test_DynamicTool(model, session, workflow_step): cls = model.DynamicTool assert cls.__tablename__ == 'dynamic_tool' @@ -385,6 +401,12 @@ def page(model, session, user): yield from dbcleanup_wrapper(session, p) +@pytest.fixture +def role(model, session): + r = model.Role() + yield from dbcleanup_wrapper(session, r) + + @pytest.fixture def user(model, session): u = model.User(email='test@example.com', password='password') From fb5a0f7a7f4d071328bb557ae2ddd3287f4165c3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 11 Jun 2021 01:50:19 -0400 Subject: [PATCH 018/225] Map DatasetPermissions declaratively --- lib/galaxy/model/__init__.py | 13 ++++++++++++- lib/galaxy/model/mapping.py | 15 +-------------- test/unit/model/test_model.py | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1d407a4168e..9adf64b7f34 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2369,6 +2369,7 @@ class Role(Base, Dictifiable, RepresentById): description = Column('description', TEXT) type = Column('type', String(40), index=True) deleted = Column('deleted', Boolean, index=True, default=False) + dataset_actions = relationship('DatasetPermissions', back_populates='role') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'name', 'description', 'type'] @@ -2462,7 +2463,17 @@ class DefaultQuotaAssociation(Dictifiable, RepresentById): self.quota = quota -class DatasetPermissions(RepresentById): +class DatasetPermissions(Base, RepresentById): + __tablename__ = 'dataset_permissions' + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + action = Column('action', TEXT) + dataset_id = Column('dataset_id', Integer, ForeignKey('dataset.id'), index=True) + role_id = Column('role_id', Integer, ForeignKey("role.id"), index=True) + dataset = relationship('Dataset', back_populates='actions') + role = relationship('Role', back_populates='dataset_actions') + def __init__(self, action, dataset, role=None, role_id=None): self.action = action self.dataset = dataset diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f680524cbca..9a91e37ed03 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -280,15 +280,6 @@ model.DefaultQuotaAssociation.table = Table( Column("type", String(32), index=True, unique=True), Column("quota_id", Integer, ForeignKey("quota.id"), index=True)) -model.DatasetPermissions.table = Table( - "dataset_permissions", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("action", TEXT), - Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), - Column("role_id", Integer, ForeignKey("role.id"), index=True)) - model.LibraryPermissions.table = Table( "library_permissions", metadata, Column("id", Integer, primary_key=True), @@ -1528,6 +1519,7 @@ simple_mapping(model.HistoryDatasetAssociation, ) simple_mapping(model.Dataset, + actions=relation(model.DatasetPermissions, back_populates='dataset'), job=relation(model.Job, primaryjoin=(model.Dataset.table.c.job_id == model.Job.table.c.id)), active_history_associations=relation(model.HistoryDatasetAssociation, primaryjoin=( @@ -1751,11 +1743,6 @@ mapper_registry.map_imperatively(model.DefaultQuotaAssociation, model.DefaultQuo quota=relation(model.Quota, backref="default") )) -mapper_registry.map_imperatively(model.DatasetPermissions, model.DatasetPermissions.table, properties=dict( - dataset=relation(model.Dataset, backref="actions"), - role=relation(model.Role, backref="dataset_actions") -)) - mapper_registry.map_imperatively(model.LibraryPermissions, model.LibraryPermissions.table, properties=dict( library=relation(model.Library, backref="actions"), role=relation(model.Role, backref="library_actions") diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 920a9927443..ae5b6a452a2 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -82,6 +82,22 @@ def test_CustosAuthnzToken(model, session, user): assert stored_obj.user == user +def test_DatasetPermissions(model, session, dataset, role): + cls = model.DatasetPermissions + assert cls.__tablename__ == 'dataset_permissions' + with dbcleanup(session, cls): + action = 'a' + obj = cls(action, dataset, role) + obj_id = persist(session, obj) + + stmt = select(cls).filter(cls.id == obj_id) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id == obj_id + assert stored_obj.action == action + assert stored_obj.dataset == dataset + assert stored_obj.role == role + + def test_DefaultUserPermissions(model, session, user, role): cls = model.DefaultUserPermissions assert cls.__tablename__ == 'default_user_permissions' @@ -388,6 +404,12 @@ def cloud_authz(model, session, user, user_authnz_token): yield from dbcleanup_wrapper(session, ca) +@pytest.fixture +def dataset(model, session): + d = model.Dataset() + yield from dbcleanup_wrapper(session, d) + + @pytest.fixture def galaxy_session(model, session, user): s = model.GalaxySession() From ca410299b2d53065c509d253f48a500c0a8c38aa Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 11 Jun 2021 02:14:51 -0400 Subject: [PATCH 019/225] Map GroupRoleAssociation declaratively --- lib/galaxy/model/__init__.py | 26 ++++++++++++++++++++------ lib/galaxy/model/mapping.py | 13 ------------- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9adf64b7f34..80d34a68513 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1895,11 +1895,12 @@ class DeferredJob(RepresentById): class Group(Base, Dictifiable, RepresentById): __tablename__ = 'galaxy_group' - id = Column("id", Integer, primary_key=True) - create_time = Column("create_time", DateTime, default=now) - update_time = Column("update_time", DateTime, default=now, onupdate=now) - name = Column("name", String(255), index=True, unique=True) - deleted = Column("deleted", Boolean, index=True, default=False) + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + name = Column('name', String(255), index=True, unique=True) + deleted = Column('deleted', Boolean, index=True, default=False) + roles = relationship('GroupRoleAssociation', back_populates='group') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'name'] @@ -2353,7 +2354,17 @@ class UserRoleAssociation(RepresentById): self.role = role -class GroupRoleAssociation(RepresentById): +class GroupRoleAssociation(Base, RepresentById): + __tablename__ = 'group_role_association' + + id = Column('id', Integer, primary_key=True) + group_id = Column('group_id', Integer, ForeignKey('galaxy_group.id'), index=True) + role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + group = relationship('Group', back_populates='roles') + role = relationship('Role', back_populates='groups') + def __init__(self, group, role): self.group = group self.role = role @@ -2370,6 +2381,7 @@ class Role(Base, Dictifiable, RepresentById): type = Column('type', String(40), index=True) deleted = Column('deleted', Boolean, index=True, default=False) dataset_actions = relationship('DatasetPermissions', back_populates='role') + groups = relationship('GroupRoleAssociation', back_populates='role') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'name', 'description', 'type'] @@ -2465,6 +2477,7 @@ class DefaultQuotaAssociation(Dictifiable, RepresentById): class DatasetPermissions(Base, RepresentById): __tablename__ = 'dataset_permissions' + id = Column('id', Integer, primary_key=True) create_time = Column('create_time', DateTime, default=now) update_time = Column('update_time', DateTime, default=now, onupdate=now) @@ -2525,6 +2538,7 @@ class LibraryDatasetDatasetAssociationPermissions(RepresentById): class DefaultUserPermissions(Base, RepresentById): __tablename__ = 'default_user_permissions' + id = Column('id', Integer, primary_key=True) user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) action = Column('action', TEXT) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 9a91e37ed03..b62284d8e10 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -248,14 +248,6 @@ model.UserRoleAssociation.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now)) -model.GroupRoleAssociation.table = Table( - "group_role_association", metadata, - Column("id", Integer, primary_key=True), - Column("group_id", Integer, ForeignKey("galaxy_group.id"), index=True), - Column("role_id", Integer, ForeignKey("role.id"), index=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now)) - model.UserQuotaAssociation.table = Table( "user_quota_association", metadata, Column("id", Integer, primary_key=True), @@ -1724,11 +1716,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.GroupRoleAssociation, model.GroupRoleAssociation.table, properties=dict( - group=relation(model.Group, backref="roles"), - role=relation(model.Role, backref="groups") -)) - mapper_registry.map_imperatively(model.UserQuotaAssociation, model.UserQuotaAssociation.table, properties=dict( user=relation(model.User, backref="quotas"), quota=relation(model.Quota, backref="users") diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index ae5b6a452a2..109b199d8f7 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -147,6 +147,20 @@ def test_DynamicTool(model, session, workflow_step): assert workflow_step in stored_obj.workflow_steps +def test_GroupRoleAssociation(model, session, group, role): + cls = model.GroupRoleAssociation + assert cls.__tablename__ == 'group_role_association' + with dbcleanup(session, cls): + obj = cls(group, role) + obj_id = persist(session, obj) + + stmt = select(cls).filter(cls.id == obj_id) + stored_obj = session.execute(stmt).scalar_one() + assert stored_obj.id == obj_id + assert stored_obj.group == group + assert stored_obj.role == role + + def test_PageRevision(model, session, page): cls = model.PageRevision assert cls.__tablename__ == 'page_revision' @@ -416,6 +430,12 @@ def galaxy_session(model, session, user): yield from dbcleanup_wrapper(session, s) +@pytest.fixture +def group(model, session): + g = model.Group() + yield from dbcleanup_wrapper(session, g) + + @pytest.fixture def page(model, session, user): p = model.Page() From f60a0f8c6364726efb2e960e230c319d7b82768c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 11 Jun 2021 02:25:55 -0400 Subject: [PATCH 020/225] Get rid of some duplicate code in tests --- test/unit/model/test_model.py | 62 ++++++++++++++--------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 109b199d8f7..1a7231701a8 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -20,8 +20,7 @@ def testAPIKeys(model, session, user): obj = cls(user_id, key) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.user_id == user_id assert stored_obj.key == key @@ -36,8 +35,7 @@ def test_CloudAuthz(model, session, user, user_authnz_token): obj = cls(user.id, provider, config, user_authnz_token.id, description) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.user_id == user.id assert stored_obj.provider == provider @@ -69,8 +67,7 @@ def test_CustosAuthnzToken(model, session, user): expiration_time, refresh_expiration_time) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.external_user_id == external_user_id assert stored_obj.provider == provider @@ -90,8 +87,7 @@ def test_DatasetPermissions(model, session, dataset, role): obj = cls(action, dataset, role) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.action == action assert stored_obj.dataset == dataset @@ -106,8 +102,7 @@ def test_DefaultUserPermissions(model, session, user, role): obj = cls(user, action, role) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.user == user assert stored_obj.action == action @@ -132,8 +127,7 @@ def test_DynamicTool(model, session, workflow_step): obj.workflow_steps.append(workflow_step) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.tool_format == tool_format assert stored_obj.tool_id == tool_id @@ -154,8 +148,7 @@ def test_GroupRoleAssociation(model, session, group, role): obj = cls(group, role) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.group == group assert stored_obj.role == role @@ -169,8 +162,7 @@ def test_PageRevision(model, session, page): obj.page = page obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time @@ -189,8 +181,7 @@ def test_PSAAssociation(model, session): obj = cls(server_url, handle, secret, issued, lifetime, assoc_type) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.server_url == server_url assert stored_obj.handle == handle @@ -209,8 +200,7 @@ def test_PSACode(model, session): obj = cls(email, code) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.email == email assert stored_obj.code == code @@ -224,8 +214,7 @@ def test_PSANonce(model, session): obj = cls(server_url, timestamp, salt) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.server_url assert stored_obj.timestamp == timestamp @@ -240,8 +229,7 @@ def test_PSAPartial(model, session): obj = cls(token, data, next_step, backend) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.token == token assert stored_obj.data == data @@ -257,8 +245,7 @@ def test_Quota(model, session): obj = cls(name, description) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time @@ -277,8 +264,7 @@ def test_Role(model, session): obj = cls(name, description) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time @@ -296,8 +282,7 @@ def testUserAction(model, session, user, galaxy_session): obj = cls(user, galaxy_session.id, action, params, context) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.user_id == user.id @@ -317,8 +302,7 @@ def test_UserAddress(model, session, user): obj = cls(user, desc, name, institution, address, city, state, postal_code, country, phone) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time @@ -347,8 +331,7 @@ def test_UserAuthnzToken(model, session, user, cloud_authz): obj.cloudauthz.append(cloud_authz) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.user_id == user.id assert stored_obj.uid == uid @@ -369,8 +352,7 @@ def test_VisualizationRevision(model, session, visualization): obj = cls(visualization, title, dbkey, config) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.create_time assert stored_obj.update_time @@ -390,8 +372,7 @@ def test_WorkerProcess(model, session): obj = cls(server_name, hostname) obj_id = persist(session, obj) - stmt = select(cls).filter(cls.id == obj_id) - stored_obj = session.execute(stmt).scalar_one() + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.server_name == server_name assert stored_obj.hostname == hostname @@ -522,6 +503,11 @@ def has_index(table, fields): return True +def get_stored_obj(session, cls, obj_id): + stmt = select(cls).filter(cls.id == obj_id) + return session.execute(stmt).scalar_one() + + def get_random_string(): """Generate unique values to accommodate unique constraints.""" return str(random.random()) From dfa9d738ec10c85709ba21224353827304df326d Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 11 Jun 2021 19:18:00 -0400 Subject: [PATCH 021/225] Map UserGroupAssociation declaratively --- lib/galaxy/model/__init__.py | 13 ++++++++++++- lib/galaxy/model/mapping.py | 14 +------------- test/unit/model/test_model.py | 13 +++++++++++++ 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 80d34a68513..3300ce6466c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1901,6 +1901,7 @@ class Group(Base, Dictifiable, RepresentById): name = Column('name', String(255), index=True, unique=True) deleted = Column('deleted', Boolean, index=True, default=False) roles = relationship('GroupRoleAssociation', back_populates='group') + users = relationship('UserGroupAssociation', back_populates='group') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'name'] @@ -1910,7 +1911,17 @@ class Group(Base, Dictifiable, RepresentById): self.deleted = False -class UserGroupAssociation(RepresentById): +class UserGroupAssociation(Base, RepresentById): + __tablename__ = 'user_group_association' + + id = Column('id', Integer, primary_key=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + group_id = Column('group_id', Integer, ForeignKey('galaxy_group.id'), index=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + user = relationship('User', back_populates='groups') + group = relationship('Group', back_populates='users') + def __init__(self, user, group): self.user = user self.group = group diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index b62284d8e10..bc96b97e391 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -232,14 +232,6 @@ model.ImplicitlyConvertedDatasetAssociation.table = Table( Column("metadata_safe", Boolean, index=True, default=True), Column("type", TrimmedString(255))) -model.UserGroupAssociation.table = Table( - "user_group_association", metadata, - Column("id", Integer, primary_key=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("group_id", Integer, ForeignKey("galaxy_group.id"), index=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now)) - model.UserRoleAssociation.table = Table( "user_role_association", metadata, Column("id", Integer, primary_key=True), @@ -1651,6 +1643,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( cloudauthz=relation(model.CloudAuthz, back_populates='user'), custos_auth=relation(model.CustosAuthnzToken, back_populates='user'), default_permissions=relation(model.DefaultUserPermissions, back_populates='user'), + groups=relation(model.UserGroupAssociation, back_populates='user'), histories=relation(model.History, backref="user", order_by=desc(model.History.update_time)), @@ -1692,11 +1685,6 @@ mapper_registry.map_imperatively(model.PasswordResetToken, model.PasswordResetTo # .preferences[pref_name] = pref_value model.User.preferences = association_proxy('_preferences', 'value', creator=model.UserPreference) # type: ignore -mapper_registry.map_imperatively(model.UserGroupAssociation, model.UserGroupAssociation.table, properties=dict( - user=relation(model.User, backref="groups"), - group=relation(model.Group, backref="users") -)) - mapper_registry.map_imperatively(model.DefaultHistoryPermissions, model.DefaultHistoryPermissions.table, properties=dict( history=relation(model.History, backref="default_permissions"), role=relation(model.Role) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 1a7231701a8..27ae0b68488 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -343,6 +343,19 @@ def test_UserAuthnzToken(model, session, user, cloud_authz): assert cloud_authz in stored_obj.cloudauthz +def test_UserGroupAssociation(model, session, user, group): + cls = model.UserGroupAssociation + assert cls.__tablename__ == 'user_group_association' + with dbcleanup(session, cls): + obj = cls(user, group) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.group == group + + def test_VisualizationRevision(model, session, visualization): cls = model.VisualizationRevision assert cls.__tablename__ == 'visualization_revision' From 12018c98f457ee3bc16d325e9593e0ce7bdad693 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 11 Jun 2021 20:10:35 -0400 Subject: [PATCH 022/225] Map UserQuotaAssociation declaratively --- lib/galaxy/model/__init__.py | 13 ++++++++++++- lib/galaxy/model/mapping.py | 14 +------------- test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3300ce6466c..27b3e08bdaf 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2412,7 +2412,17 @@ class Role(Base, Dictifiable, RepresentById): self.deleted = deleted -class UserQuotaAssociation(Dictifiable, RepresentById): +class UserQuotaAssociation(Base, Dictifiable, RepresentById): + __tablename__ = 'user_quota_association' + + id = Column('id', Integer, primary_key=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + quota_id = Column("quota_id", Integer, ForeignKey("quota.id"), index=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + user = relationship('User', back_populates='quotas') + quota = relationship('Quota', back_populates='users') + dict_element_visible_keys = ['user'] def __init__(self, user, quota): @@ -2439,6 +2449,7 @@ class Quota(Base, Dictifiable, RepresentById): bytes = Column('bytes', BigInteger) operation = Column('operation', String(8)) deleted = Column('deleted', Boolean, index=True, default=False) + users = relationship('UserQuotaAssociation', back_populates='quota') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'name', 'description', 'bytes', 'operation', 'display_amount', 'default', 'users', 'groups'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index bc96b97e391..c20b6892424 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -240,14 +240,6 @@ model.UserRoleAssociation.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now)) -model.UserQuotaAssociation.table = Table( - "user_quota_association", metadata, - Column("id", Integer, primary_key=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("quota_id", Integer, ForeignKey("quota.id"), index=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now)) - model.GroupQuotaAssociation.table = Table( "group_quota_association", metadata, Column("id", Integer, primary_key=True), @@ -1657,6 +1649,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( galaxy_sessions=relation(model.GalaxySession, backref="user", order_by=desc(model.GalaxySession.table.c.update_time)), + quotas=relation(model.UserQuotaAssociation, back_populates='user'), social_auth=relation(model.UserAuthnzToken, back_populates='user'), stored_workflow_menu_entries=relation(model.StoredWorkflowMenuEntry, primaryjoin=( @@ -1704,11 +1697,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.UserQuotaAssociation, model.UserQuotaAssociation.table, properties=dict( - user=relation(model.User, backref="quotas"), - quota=relation(model.Quota, backref="users") -)) - mapper_registry.map_imperatively(model.GroupQuotaAssociation, model.GroupQuotaAssociation.table, properties=dict( group=relation(model.Group, backref="quotas"), quota=relation(model.Quota, backref="groups") diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 27ae0b68488..2b304b3fd4c 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -356,6 +356,19 @@ def test_UserGroupAssociation(model, session, user, group): assert stored_obj.group == group +def test_UserQuotaAssociation(model, session, user, quota): + cls = model.UserQuotaAssociation + assert cls.__tablename__ == 'user_quota_association' + with dbcleanup(session, cls): + obj = cls(user, quota) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.quota == quota + + def test_VisualizationRevision(model, session, visualization): cls = model.VisualizationRevision assert cls.__tablename__ == 'visualization_revision' @@ -437,6 +450,12 @@ def page(model, session, user): yield from dbcleanup_wrapper(session, p) +@pytest.fixture +def quota(model, session): + q = model.Quota(get_random_string(), 'b') + yield from dbcleanup_wrapper(session, q) + + @pytest.fixture def role(model, session): r = model.Role() From 06df91c00fc128fbf3f1a5117b8a973915ae9d57 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 11 Jun 2021 20:20:55 -0400 Subject: [PATCH 023/225] Map GroupQuotaAssociation declaratively --- lib/galaxy/model/__init__.py | 16 ++++++++++++++-- lib/galaxy/model/mapping.py | 13 ------------- test/unit/model/test_model.py | 13 +++++++++++++ 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 27b3e08bdaf..5915dd2d897 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1900,6 +1900,7 @@ class Group(Base, Dictifiable, RepresentById): update_time = Column('update_time', DateTime, default=now, onupdate=now) name = Column('name', String(255), index=True, unique=True) deleted = Column('deleted', Boolean, index=True, default=False) + quotas = relationship('GroupQuotaAssociation', back_populates='group') roles = relationship('GroupRoleAssociation', back_populates='group') users = relationship('UserGroupAssociation', back_populates='group') @@ -2417,7 +2418,7 @@ class UserQuotaAssociation(Base, Dictifiable, RepresentById): id = Column('id', Integer, primary_key=True) user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - quota_id = Column("quota_id", Integer, ForeignKey("quota.id"), index=True) + quota_id = Column('quota_id', Integer, ForeignKey('quota.id'), index=True) create_time = Column('create_time', DateTime, default=now) update_time = Column('update_time', DateTime, default=now, onupdate=now) user = relationship('User', back_populates='quotas') @@ -2430,7 +2431,17 @@ class UserQuotaAssociation(Base, Dictifiable, RepresentById): self.quota = quota -class GroupQuotaAssociation(Dictifiable, RepresentById): +class GroupQuotaAssociation(Base, Dictifiable, RepresentById): + __tablename__ = 'group_quota_association' + + id = Column('id', Integer, primary_key=True) + group_id = Column("group_id", Integer, ForeignKey("galaxy_group.id"), index=True) + quota_id = Column('quota_id', Integer, ForeignKey('quota.id'), index=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + group = relationship('Group', back_populates='quotas') + quota = relationship('Quota', back_populates='groups') + dict_element_visible_keys = ['group'] def __init__(self, group, quota): @@ -2449,6 +2460,7 @@ class Quota(Base, Dictifiable, RepresentById): bytes = Column('bytes', BigInteger) operation = Column('operation', String(8)) deleted = Column('deleted', Boolean, index=True, default=False) + groups = relationship('GroupQuotaAssociation', back_populates='quota') users = relationship('UserQuotaAssociation', back_populates='quota') dict_collection_visible_keys = ['id', 'name'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index c20b6892424..b64f7776b35 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -240,14 +240,6 @@ model.UserRoleAssociation.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now)) -model.GroupQuotaAssociation.table = Table( - "group_quota_association", metadata, - Column("id", Integer, primary_key=True), - Column("group_id", Integer, ForeignKey("galaxy_group.id"), index=True), - Column("quota_id", Integer, ForeignKey("quota.id"), index=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now)) - model.DefaultQuotaAssociation.table = Table( "default_quota_association", metadata, Column("id", Integer, primary_key=True), @@ -1697,11 +1689,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.GroupQuotaAssociation, model.GroupQuotaAssociation.table, properties=dict( - group=relation(model.Group, backref="quotas"), - quota=relation(model.Quota, backref="groups") -)) - mapper_registry.map_imperatively(model.DefaultQuotaAssociation, model.DefaultQuotaAssociation.table, properties=dict( quota=relation(model.Quota, backref="default") )) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 2b304b3fd4c..26df99fa58e 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -141,6 +141,19 @@ def test_DynamicTool(model, session, workflow_step): assert workflow_step in stored_obj.workflow_steps +def test_GroupQuotaAssociation(model, session, group, quota): + cls = model.GroupQuotaAssociation + assert cls.__tablename__ == 'group_quota_association' + with dbcleanup(session, cls): + obj = cls(group, quota) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.group == group + assert stored_obj.quota == quota + + def test_GroupRoleAssociation(model, session, group, role): cls = model.GroupRoleAssociation assert cls.__tablename__ == 'group_role_association' From 37edac433fb879a82f37dd8e4069e3dc938083b2 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 11 Jun 2021 20:38:40 -0400 Subject: [PATCH 024/225] Map DefaultQuotaAssociation declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 12 ------------ lib/galaxy/quota/__init__.py | 4 ++-- test/unit/model/test_model.py | 14 ++++++++++++++ 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 5915dd2d897..f5084b61693 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2460,6 +2460,7 @@ class Quota(Base, Dictifiable, RepresentById): bytes = Column('bytes', BigInteger) operation = Column('operation', String(8)) deleted = Column('deleted', Boolean, index=True, default=False) + default = relationship('DefaultQuotaAssociation', back_populates='quota') groups = relationship('GroupQuotaAssociation', back_populates='quota') users = relationship('UserQuotaAssociation', back_populates='quota') @@ -2496,7 +2497,16 @@ class Quota(Base, Dictifiable, RepresentById): return galaxy.util.nice_size(self.bytes) -class DefaultQuotaAssociation(Dictifiable, RepresentById): +class DefaultQuotaAssociation(Base, Dictifiable, RepresentById): + __tablename__ = 'default_quota_association' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + type = Column('type', String(32), index=True, unique=True) + quota_id = Column('quota_id', Integer, ForeignKey('quota.id'), index=True) + quota = relationship('Quota', back_populates='default') + dict_element_visible_keys = ['type'] class types(str, Enum): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index b64f7776b35..ff00a39862f 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -240,14 +240,6 @@ model.UserRoleAssociation.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now)) -model.DefaultQuotaAssociation.table = Table( - "default_quota_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("type", String(32), index=True, unique=True), - Column("quota_id", Integer, ForeignKey("quota.id"), index=True)) - model.LibraryPermissions.table = Table( "library_permissions", metadata, Column("id", Integer, primary_key=True), @@ -1689,10 +1681,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.DefaultQuotaAssociation, model.DefaultQuotaAssociation.table, properties=dict( - quota=relation(model.Quota, backref="default") -)) - mapper_registry.map_imperatively(model.LibraryPermissions, model.LibraryPermissions.table, properties=dict( library=relation(model.Library, backref="actions"), role=relation(model.Role, backref="library_actions") diff --git a/lib/galaxy/quota/__init__.py b/lib/galaxy/quota/__init__.py index d1ca9873593..baa8101cb53 100644 --- a/lib/galaxy/quota/__init__.py +++ b/lib/galaxy/quota/__init__.py @@ -144,7 +144,7 @@ class DatabaseQuotaAgent(QuotaAgent): return self._default_quota(self.model.DefaultQuotaAssociation.types.REGISTERED) def _default_quota(self, default_type): - dqa = self.sa_session.query(self.model.DefaultQuotaAssociation).filter(self.model.DefaultQuotaAssociation.table.c.type == default_type).first() + dqa = self.sa_session.query(self.model.DefaultQuotaAssociation).filter(self.model.DefaultQuotaAssociation.type == default_type).first() if not dqa: return None if dqa.quota.bytes < 0: @@ -161,7 +161,7 @@ class DatabaseQuotaAgent(QuotaAgent): for gqa in quota.groups: self.sa_session.delete(gqa) # Find the old default, assign the new quota if it exists - dqa = self.sa_session.query(self.model.DefaultQuotaAssociation).filter(self.model.DefaultQuotaAssociation.table.c.type == default_type).first() + dqa = self.sa_session.query(self.model.DefaultQuotaAssociation).filter(self.model.DefaultQuotaAssociation.type == default_type).first() if dqa: dqa.quota = quota # Or create if necessary diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 26df99fa58e..731484bb527 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -94,6 +94,20 @@ def test_DatasetPermissions(model, session, dataset, role): assert stored_obj.role == role +def test_DefaultQuotaAssociation(model, session, quota): + cls = model.DefaultQuotaAssociation + assert cls.__tablename__ == 'default_quota_association' + with dbcleanup(session, cls): + type = cls.types.REGISTERED + obj = cls(type, quota) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.type == type + assert stored_obj.quota == quota + + def test_DefaultUserPermissions(model, session, user, role): cls = model.DefaultUserPermissions assert cls.__tablename__ == 'default_user_permissions' From bc174c4fb41b07bc53c6cc4a6ac808d8e72859e8 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 11 Jun 2021 21:09:56 -0400 Subject: [PATCH 025/225] Map PageUserShareAssociation declaratively --- lib/galaxy/model/__init__.py | 10 +++++++++- lib/galaxy/model/mapping.py | 14 +++----------- test/unit/model/test_model.py | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index f5084b61693..9e44a9ad66b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6637,7 +6637,15 @@ class PageRevision(Base, Dictifiable, RepresentById): return rval -class PageUserShareAssociation(UserShareAssociation): +class PageUserShareAssociation(Base, UserShareAssociation): + __tablename__ = 'page_user_share_association' + + id = Column("id", Integer, primary_key=True) + page_id = Column("page_id", Integer, ForeignKey("page.id"), index=True) + user_id = Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True) + user = relationship('User', back_populates='pages_shared_by_others') + page = relationship('Page', back_populates='users_shared_with') + def __init__(self): self.page = None self.user = None diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index ff00a39862f..bf730ea1c3b 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1062,12 +1062,6 @@ model.Page.table = Table( Index('ix_page_slug', 'slug', mysql_length=200), ) -model.PageUserShareAssociation.table = Table( - "page_user_share_association", metadata, - Column("id", Integer, primary_key=True), - Column("page_id", Integer, ForeignKey("page.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True)) - model.Visualization.table = Table( "visualization", metadata, Column("id", Integer, primary_key=True), @@ -1633,6 +1627,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( galaxy_sessions=relation(model.GalaxySession, backref="user", order_by=desc(model.GalaxySession.table.c.update_time)), + pages_shared_by_others=relation(model.PageUserShareAssociation, back_populates='user'), quotas=relation(model.UserQuotaAssociation, back_populates='user'), social_auth=relation(model.UserAuthnzToken, back_populates='user'), stored_workflow_menu_entries=relation(model.StoredWorkflowMenuEntry, @@ -2309,7 +2304,8 @@ mapper_registry.map_imperatively(model.Page, model.Page.table, properties=dict( average_rating=column_property( select(func.avg(model.PageRatingAssociation.table.c.rating)).where(model.PageRatingAssociation.table.c.page_id == model.Page.table.c.id).scalar_subquery(), deferred=True - ) + ), + users_shared_with=relation(model.PageUserShareAssociation, back_populates='page') )) # Set up proxy so that @@ -2317,10 +2313,6 @@ mapper_registry.map_imperatively(model.Page, model.Page.table, properties=dict( # returns a list of users that page is shared with. model.Page.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore -mapper_registry.map_imperatively(model.PageUserShareAssociation, model.PageUserShareAssociation.table, - properties=dict(user=relation(model.User, backref='pages_shared_by_others'), - page=relation(model.Page, backref='users_shared_with'))) - mapper_registry.map_imperatively(model.Visualization, model.Visualization.table, properties=dict( user=relation(model.User), revisions=relation(model.VisualizationRevision, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 731484bb527..322c36100f0 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -200,6 +200,21 @@ def test_PageRevision(model, session, page): assert stored_obj.page.id == page.id +def test_PageUserShareAssociation(model, session, page, user): + cls = model.PageUserShareAssociation + assert cls.__tablename__ == 'page_user_share_association' + with dbcleanup(session, cls): + obj = cls() + obj.page = page + obj.user = user + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.page == page + assert stored_obj.user == user + + def test_PSAAssociation(model, session): cls = model.PSAAssociation assert cls.__tablename__ == 'psa_association' From 050c5ebee83e2f009da8baccb64fb593d136d60d Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 15 Jun 2021 19:07:47 -0400 Subject: [PATCH 026/225] Simplify rating classes initialization (see note) 1. Remove id parameter from constructor (this is created by the db). 2. Remove `item` instance attribute: it's not used and is meaningless. 3. Make user and item parameters in the constructor required: a rating cannot exist without a rater or the item being rated. 4. For the same reason as in #2, make the `set_item()` method private and only call it from the constructor. 5. Simplify mapping test: just use the constructor. --- lib/galaxy/managers/ratable.py | 2 +- lib/galaxy/model/__init__.py | 22 +++++++++--------- lib/galaxy/model/item_attrs.py | 5 +---- test/unit/data/test_galaxy_mapping.py | 32 +++++++++++++-------------- 4 files changed, 28 insertions(+), 33 deletions(-) diff --git a/lib/galaxy/managers/ratable.py b/lib/galaxy/managers/ratable.py index b1bf4a08e67..b86c5505e3d 100644 --- a/lib/galaxy/managers/ratable.py +++ b/lib/galaxy/managers/ratable.py @@ -49,7 +49,7 @@ class RatableManagerMixin: # TODO?: update and create to RatingsManager (if not overkill) rating = self.rating(item, user, as_int=False) if not rating: - rating = self.rating_assoc(user=user) + rating = self.rating_assoc(user, item) self.associate(rating, item) rating.rating = value diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9e44a9ad66b..b9f160f36fa 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6874,48 +6874,48 @@ class LibraryDatasetCollectionAnnotationAssociation(RepresentById): # Item rating classes. class ItemRatingAssociation: - def __init__(self, id=None, user=None, item=None, rating=0): - self.id = id + def __init__(self, user, item, rating=0): self.user = user - self.item = item self.rating = rating + self._set_item(item) - def set_item(self, item): + def _set_item(self, item): """ Set association's item. """ + raise NotImplementedError() class HistoryRatingAssociation(ItemRatingAssociation, RepresentById): - def set_item(self, history): + def _set_item(self, history): self.history = history class HistoryDatasetAssociationRatingAssociation(ItemRatingAssociation, RepresentById): - def set_item(self, history_dataset_association): + def _set_item(self, history_dataset_association): self.history_dataset_association = history_dataset_association class StoredWorkflowRatingAssociation(ItemRatingAssociation, RepresentById): - def set_item(self, stored_workflow): + def _set_item(self, stored_workflow): self.stored_workflow = stored_workflow class PageRatingAssociation(ItemRatingAssociation, RepresentById): - def set_item(self, page): + def _set_item(self, page): self.page = page class VisualizationRatingAssociation(ItemRatingAssociation, RepresentById): - def set_item(self, visualization): + def _set_item(self, visualization): self.visualization = visualization class HistoryDatasetCollectionRatingAssociation(ItemRatingAssociation, RepresentById): - def set_item(self, dataset_collection): + def _set_item(self, dataset_collection): self.dataset_collection = dataset_collection class LibraryDatasetCollectionRatingAssociation(ItemRatingAssociation, RepresentById): - def set_item(self, dataset_collection): + def _set_item(self, dataset_collection): self.dataset_collection = dataset_collection diff --git a/lib/galaxy/model/item_attrs.py b/lib/galaxy/model/item_attrs.py index 473b3c77ae9..fde16e71515 100644 --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -43,10 +43,7 @@ class UsesItemRatings: if not item_rating: # User has not yet rated item; create rating. item_rating_assoc_class = self._get_item_rating_assoc_class(item, webapp_model=webapp_model) - item_rating = item_rating_assoc_class() - item_rating.user = user - item_rating.set_item(item) - item_rating.rating = rating + item_rating = item_rating_assoc_class(user, item, rating) db_session.add(item_rating) db_session.flush() elif item_rating.rating != rating: diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py index 430b333dc0a..27b9b3a0a12 100644 --- a/test/unit/data/test_galaxy_mapping.py +++ b/test/unit/data/test_galaxy_mapping.py @@ -130,52 +130,50 @@ class MappingTests(BaseModelTestCase): def test_ratings(self): model = self.model - u = model.User(email="rater@example.com", password="password") + user_email = "rater@example.com" + u = model.User(email=user_email, password="password") self.persist(u) - def persist_and_check_rating(rating_class, **kwds): - rating_association = rating_class() - rating_association.rating = 5 - rating_association.user = u - for key, value in kwds.items(): - setattr(rating_association, key, value) + def persist_and_check_rating(rating_class, item): + rating = 5 + rating_association = rating_class(u, item, rating) self.persist(rating_association) self.expunge() - stored_annotation = self.query(rating_class).all()[0] - assert stored_annotation.rating == 5 - assert stored_annotation.user.email == "rater@example.com" + stored_rating = self.query(rating_class).all()[0] + assert stored_rating.rating == rating + assert stored_rating.user.email == user_email sw = model.StoredWorkflow() sw.user = u self.persist(sw) - persist_and_check_rating(model.StoredWorkflowRatingAssociation, stored_workflow=sw) + persist_and_check_rating(model.StoredWorkflowRatingAssociation, sw) h = model.History(name="History for Rating", user=u) self.persist(h) - persist_and_check_rating(model.HistoryRatingAssociation, history=h) + persist_and_check_rating(model.HistoryRatingAssociation, h) d1 = model.HistoryDatasetAssociation(extension="txt", history=h, create_dataset=True, sa_session=model.session) self.persist(d1) - persist_and_check_rating(model.HistoryDatasetAssociationRatingAssociation, hda=d1) + persist_and_check_rating(model.HistoryDatasetAssociationRatingAssociation, d1) page = model.Page() page.user = u self.persist(page) - persist_and_check_rating(model.PageRatingAssociation, page=page) + persist_and_check_rating(model.PageRatingAssociation, page) visualization = model.Visualization() visualization.user = u self.persist(visualization) - persist_and_check_rating(model.VisualizationRatingAssociation, visualization=visualization) + persist_and_check_rating(model.VisualizationRatingAssociation, visualization) dataset_collection = model.DatasetCollection(collection_type="paired") history_dataset_collection = model.HistoryDatasetCollectionAssociation(collection=dataset_collection) self.persist(history_dataset_collection) - persist_and_check_rating(model.HistoryDatasetCollectionRatingAssociation, history_dataset_collection=history_dataset_collection) + persist_and_check_rating(model.HistoryDatasetCollectionRatingAssociation, history_dataset_collection) library_dataset_collection = model.LibraryDatasetCollectionAssociation(collection=dataset_collection) self.persist(library_dataset_collection) - persist_and_check_rating(model.LibraryDatasetCollectionRatingAssociation, library_dataset_collection=library_dataset_collection) + persist_and_check_rating(model.LibraryDatasetCollectionRatingAssociation, library_dataset_collection) def test_display_name(self): From 3aeb513585562a7a2e2afb24dfa0ed75818ce9f6 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 15 Jun 2021 23:03:46 -0400 Subject: [PATCH 027/225] Map HistoryRatingAssociation declaratively (see note) Note: tests will break: ItemRatingAssociation is the base class for all rating association models. Now that it's mapped using declarative mapping, it inherits from the declarative base class, Base. Which is why any subclass of ItemRatingAssociation MUST CONTAIN A TABLE AND A PRIMARY KEY COLUMN - but the other rating association models have not been migrated to declarative yet (coming up in subsequent commits). NOT inheriting from Base and, instead, having the concrete subclasses inherit from Base will solve this issue, but will cause another: declarative base WILL OVERRIDE THE CONSTRUCTOR specified in a parent class; so the __init__ method specified in ItemRatingAssociation will NOT be called - which will break other code. --- lib/galaxy/model/__init__.py | 13 ++++++++++++- lib/galaxy/model/mapping.py | 13 +++---------- test/unit/model/test_model.py | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b9f160f36fa..950f92a1d37 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6873,7 +6873,9 @@ class LibraryDatasetCollectionAnnotationAssociation(RepresentById): # Item rating classes. -class ItemRatingAssociation: +class ItemRatingAssociation(Base): + __abstract__ = True + def __init__(self, user, item, rating=0): self.user = user self.rating = rating @@ -6885,6 +6887,15 @@ class ItemRatingAssociation: class HistoryRatingAssociation(ItemRatingAssociation, RepresentById): + __tablename__ = 'history_rating_association' + + id = Column('id', Integer, primary_key=True) + history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column('rating', Integer, index=True) + history = relationship('History', back_populates='ratings') + user = relationship('User') + def _set_item(self, history): self.history = history diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index bf730ea1c3b..1f1ed5a1973 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1293,12 +1293,6 @@ model.LibraryDatasetCollectionAnnotationAssociation.table = Table( ) # Ratings tables. -model.HistoryRatingAssociation.table = Table("history_rating_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("rating", Integer, index=True)) - model.HistoryDatasetAssociationRatingAssociation.table = Table( "history_dataset_association_rating_association", metadata, Column("id", Integer, primary_key=True), @@ -1581,10 +1575,10 @@ mapper_registry.map_imperatively(model.History, model.History.table, properties= order_by=model.HistoryAnnotationAssociation.table.c.id, backref="history"), ratings=relation(model.HistoryRatingAssociation, - order_by=model.HistoryRatingAssociation.table.c.id, - backref="history"), + order_by=model.HistoryRatingAssociation.id, + back_populates="history"), average_rating=column_property( - select(func.avg(model.HistoryRatingAssociation.table.c.rating)).where(model.HistoryRatingAssociation.table.c.history_id == model.History.table.c.id).scalar_subquery(), + select(func.avg(model.HistoryRatingAssociation.rating)).where(model.HistoryRatingAssociation.history_id == model.History.table.c.id).scalar_subquery(), deferred=True ), users_shared_with_count=column_property( @@ -2393,7 +2387,6 @@ def rating_mapping(rating_class, **kwds): simple_mapping(rating_class, **dict(user=relation(model.User), **kwds)) -rating_mapping(model.HistoryRatingAssociation) rating_mapping(model.HistoryDatasetAssociationRatingAssociation) rating_mapping(model.StoredWorkflowRatingAssociation) rating_mapping(model.PageRatingAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 322c36100f0..61f2c717db8 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -181,6 +181,21 @@ def test_GroupRoleAssociation(model, session, group, role): assert stored_obj.role == role +def test_HistoryRatingAssociation(model, session, history, user): + cls = model.HistoryRatingAssociation + assert cls.__tablename__ == 'history_rating_association' + with dbcleanup(session, cls): + rating = 9 + obj = cls(user, history, rating) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.history == history + assert stored_obj.rating == rating + + def test_PageRevision(model, session, page): cls = model.PageRevision assert cls.__tablename__ == 'page_revision' @@ -467,6 +482,12 @@ def cloud_authz(model, session, user, user_authnz_token): yield from dbcleanup_wrapper(session, ca) +@pytest.fixture +def history(model, session): + h = model.History() + yield from dbcleanup_wrapper(session, h) + + @pytest.fixture def dataset(model, session): d = model.Dataset() From 5db90340ef9565ae93499cb5520c9bc1e1ca80cb Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 15 Jun 2021 23:23:51 -0400 Subject: [PATCH 028/225] Temporarily fix tests for not yet mapped rating assoc models Because we need these tests functioning for remapping these models. --- lib/galaxy/model/__init__.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 950f92a1d37..edfbbae08a8 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6886,6 +6886,20 @@ class ItemRatingAssociation(Base): raise NotImplementedError() +class _remove_ItemRatingAssociation: + # This is a temporary fix to make the yet-to-be-mapped-declaratively rating + # association models pass the tests. TO BE REMOVED BEFORE REVIEW. + + def __init__(self, user, item, rating=0): + self.user = user + self.rating = rating + self._set_item(item) + + def _set_item(self, item): + """ Set association's item. """ + raise NotImplementedError() + + class HistoryRatingAssociation(ItemRatingAssociation, RepresentById): __tablename__ = 'history_rating_association' @@ -6900,32 +6914,32 @@ class HistoryRatingAssociation(ItemRatingAssociation, RepresentById): self.history = history -class HistoryDatasetAssociationRatingAssociation(ItemRatingAssociation, RepresentById): +class HistoryDatasetAssociationRatingAssociation(_remove_ItemRatingAssociation, RepresentById): def _set_item(self, history_dataset_association): self.history_dataset_association = history_dataset_association -class StoredWorkflowRatingAssociation(ItemRatingAssociation, RepresentById): +class StoredWorkflowRatingAssociation(_remove_ItemRatingAssociation, RepresentById): def _set_item(self, stored_workflow): self.stored_workflow = stored_workflow -class PageRatingAssociation(ItemRatingAssociation, RepresentById): +class PageRatingAssociation(_remove_ItemRatingAssociation, RepresentById): def _set_item(self, page): self.page = page -class VisualizationRatingAssociation(ItemRatingAssociation, RepresentById): +class VisualizationRatingAssociation(_remove_ItemRatingAssociation, RepresentById): def _set_item(self, visualization): self.visualization = visualization -class HistoryDatasetCollectionRatingAssociation(ItemRatingAssociation, RepresentById): +class HistoryDatasetCollectionRatingAssociation(_remove_ItemRatingAssociation, RepresentById): def _set_item(self, dataset_collection): self.dataset_collection = dataset_collection -class LibraryDatasetCollectionRatingAssociation(ItemRatingAssociation, RepresentById): +class LibraryDatasetCollectionRatingAssociation(_remove_ItemRatingAssociation, RepresentById): def _set_item(self, dataset_collection): self.dataset_collection = dataset_collection From 9d19c89a81083bcbdb34efba496fa47f22db9036 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 00:34:20 -0400 Subject: [PATCH 029/225] Fix bug (see note) + map HDARatingAssoc declaratively BUG: In the previous version, HDARatingAssoc was never set. The attribute that was set was `history_dataset_association`. However, this value was never translated by SQLAlchemy into a relationship because there is not connection between this attribute and the `history_dataset_association_id` foreign key. Thus, a relationship was NOT added upon session flush. The relationship that was defined was `hda` - defined in mapping.py under HistoryDatasetAssociation mapping via backref attribute. But this is covered by a unit test in test/unit/data/test_galaxy_mapping.py: why did the test pass? Because the test was NOT doing what the code does; instead the test was using `setattr` to set the correct attribute in the test itself. So the test avoided the bug via an alternative implementation of the code under test. In my option, the following will help avoid such issues: 1. Use back_populates instead of backref to keep relationships explicit. 2. Keep testing code simple. --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 13 ++----------- test/unit/model/test_model.py | 21 +++++++++++++++++++++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index edfbbae08a8..20d806c7dac 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6914,7 +6914,17 @@ class HistoryRatingAssociation(ItemRatingAssociation, RepresentById): self.history = history -class HistoryDatasetAssociationRatingAssociation(_remove_ItemRatingAssociation, RepresentById): +class HistoryDatasetAssociationRatingAssociation(ItemRatingAssociation, RepresentById): + __tablename__ = 'history_dataset_association_rating_association' + + id = Column('id', Integer, primary_key=True) + history_dataset_association_id = Column('history_dataset_association_id', Integer, + ForeignKey('history_dataset_association.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column('rating', Integer, index=True) + history_dataset_association = relationship('HistoryDatasetAssociation', back_populates='ratings') + user = relationship('User') + def _set_item(self, history_dataset_association): self.history_dataset_association = history_dataset_association diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 1f1ed5a1973..f34d1b230e7 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1293,14 +1293,6 @@ model.LibraryDatasetCollectionAnnotationAssociation.table = Table( ) # Ratings tables. -model.HistoryDatasetAssociationRatingAssociation.table = Table( - "history_dataset_association_rating_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_dataset_association_id", Integer, - ForeignKey("history_dataset_association.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("rating", Integer, index=True)) - model.StoredWorkflowRatingAssociation.table = Table( "stored_workflow_rating_association", metadata, Column("id", Integer, primary_key=True), @@ -1453,8 +1445,8 @@ simple_mapping(model.HistoryDatasetAssociation, order_by=model.HistoryDatasetAssociationAnnotationAssociation.table.c.id, backref="hda"), ratings=relation(model.HistoryDatasetAssociationRatingAssociation, - order_by=model.HistoryDatasetAssociationRatingAssociation.table.c.id, - backref="hda"), + order_by=model.HistoryDatasetAssociationRatingAssociation.id, + back_populates="history_dataset_association"), extended_metadata=relation(model.ExtendedMetadata, primaryjoin=(model.HistoryDatasetAssociation.table.c.extended_metadata_id == model.ExtendedMetadata.table.c.id)), @@ -2387,7 +2379,6 @@ def rating_mapping(rating_class, **kwds): simple_mapping(rating_class, **dict(user=relation(model.User), **kwds)) -rating_mapping(model.HistoryDatasetAssociationRatingAssociation) rating_mapping(model.StoredWorkflowRatingAssociation) rating_mapping(model.PageRatingAssociation) rating_mapping(model.VisualizationRatingAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 61f2c717db8..5425d5f97f2 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -196,6 +196,21 @@ def test_HistoryRatingAssociation(model, session, history, user): assert stored_obj.rating == rating +def test_HistoryDatasetAssociationRatingAssociation(model, session, history_dataset_association, user): + cls = model.HistoryDatasetAssociationRatingAssociation + assert cls.__tablename__ == 'history_dataset_association_rating_association' + with dbcleanup(session, cls): + rating = 9 + obj = cls(user, history_dataset_association, rating) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.history_dataset_association == history_dataset_association + assert stored_obj.rating == rating + + def test_PageRevision(model, session, page): cls = model.PageRevision assert cls.__tablename__ == 'page_revision' @@ -488,6 +503,12 @@ def history(model, session): yield from dbcleanup_wrapper(session, h) +@pytest.fixture +def history_dataset_association(model, session, dataset): + hda = model.HistoryDatasetAssociation(dataset=dataset) + yield from dbcleanup_wrapper(session, hda) + + @pytest.fixture def dataset(model, session): d = model.Dataset() From a52e98b5a80bc9c95180930b3af3d2a37cd91986 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 01:17:00 -0400 Subject: [PATCH 030/225] Map StoredWorkflowRatingAssociation declaratively --- lib/galaxy/model/__init__.py | 11 ++++++++++- lib/galaxy/model/mapping.py | 14 +++----------- test/unit/model/test_model.py | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 20d806c7dac..ab462005cc8 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6929,7 +6929,16 @@ class HistoryDatasetAssociationRatingAssociation(ItemRatingAssociation, Represen self.history_dataset_association = history_dataset_association -class StoredWorkflowRatingAssociation(_remove_ItemRatingAssociation, RepresentById): +class StoredWorkflowRatingAssociation(ItemRatingAssociation, RepresentById): + __tablename__ = 'stored_workflow_rating_association' + + id = Column('id', Integer, primary_key=True) + stored_workflow_id = Column('stored_workflow_id', Integer, ForeignKey('stored_workflow.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column('rating', Integer, index=True) + stored_workflow = relationship('StoredWorkflow', back_populates='ratings') + user = relationship('User') + def _set_item(self, stored_workflow): self.stored_workflow = stored_workflow diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f34d1b230e7..e01d0f1b76c 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1293,13 +1293,6 @@ model.LibraryDatasetCollectionAnnotationAssociation.table = Table( ) # Ratings tables. -model.StoredWorkflowRatingAssociation.table = Table( - "stored_workflow_rating_association", metadata, - Column("id", Integer, primary_key=True), - Column("stored_workflow_id", Integer, ForeignKey("stored_workflow.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("rating", Integer, index=True)) - model.PageRatingAssociation.table = Table( "page_rating_association", metadata, Column("id", Integer, primary_key=True), @@ -2129,10 +2122,10 @@ mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.tabl order_by=model.StoredWorkflowAnnotationAssociation.table.c.id, backref="stored_workflow"), ratings=relation(model.StoredWorkflowRatingAssociation, - order_by=model.StoredWorkflowRatingAssociation.table.c.id, - backref="stored_workflow"), + order_by=model.StoredWorkflowRatingAssociation.id, + back_populates="stored_workflow"), average_rating=column_property( - select(func.avg(model.StoredWorkflowRatingAssociation.table.c.rating)).where(model.StoredWorkflowRatingAssociation.table.c.stored_workflow_id == model.StoredWorkflow.table.c.id).scalar_subquery(), + select(func.avg(model.StoredWorkflowRatingAssociation.rating)).where(model.StoredWorkflowRatingAssociation.stored_workflow_id == model.StoredWorkflow.table.c.id).scalar_subquery(), deferred=True ) )) @@ -2379,7 +2372,6 @@ def rating_mapping(rating_class, **kwds): simple_mapping(rating_class, **dict(user=relation(model.User), **kwds)) -rating_mapping(model.StoredWorkflowRatingAssociation) rating_mapping(model.PageRatingAssociation) rating_mapping(model.VisualizationRatingAssociation) rating_mapping(model.HistoryDatasetCollectionRatingAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 5425d5f97f2..b6746d95829 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -346,6 +346,21 @@ def test_Role(model, session): assert stored_obj.deleted is False +def test_StoredWorkflowRatingAssociation(model, session, stored_workflow, user): + cls = model.StoredWorkflowRatingAssociation + assert cls.__tablename__ == 'stored_workflow_rating_association' + with dbcleanup(session, cls): + rating = 9 + obj = cls(user, stored_workflow, rating) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.stored_workflow == stored_workflow + assert stored_obj.rating == rating + + def testUserAction(model, session, user, galaxy_session): cls = model.UserAction assert cls.__tablename__ == 'user_action' @@ -546,6 +561,13 @@ def role(model, session): yield from dbcleanup_wrapper(session, r) +@pytest.fixture +def stored_workflow(model, session, user): + w = model.StoredWorkflow() + w.user = user + yield from dbcleanup_wrapper(session, w) + + @pytest.fixture def user(model, session): u = model.User(email='test@example.com', password='password') From edef3c40189f85d52235db589d1daec68a380446 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 01:23:14 -0400 Subject: [PATCH 031/225] Map PageRatingAssociation declaratively --- lib/galaxy/model/__init__.py | 11 ++++++++++- lib/galaxy/model/mapping.py | 14 +++----------- test/unit/model/test_model.py | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ab462005cc8..ba76ae6a974 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6943,7 +6943,16 @@ class StoredWorkflowRatingAssociation(ItemRatingAssociation, RepresentById): self.stored_workflow = stored_workflow -class PageRatingAssociation(_remove_ItemRatingAssociation, RepresentById): +class PageRatingAssociation(ItemRatingAssociation, RepresentById): + __tablename__ = 'page_rating_association' + + id = Column('id', Integer, primary_key=True) + page_id = Column('page_id', Integer, ForeignKey('page.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column('rating', Integer, index=True) + page = relationship('Page', back_populates='ratings') + user = relationship('User') + def _set_item(self, page): self.page = page diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index e01d0f1b76c..9ddbef4888a 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1293,13 +1293,6 @@ model.LibraryDatasetCollectionAnnotationAssociation.table = Table( ) # Ratings tables. -model.PageRatingAssociation.table = Table( - "page_rating_association", metadata, - Column("id", Integer, primary_key=True), - Column("page_id", Integer, ForeignKey("page.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("rating", Integer, index=True)) - model.VisualizationRatingAssociation.table = Table( "visualization_rating_association", metadata, Column("id", Integer, primary_key=True), @@ -2278,10 +2271,10 @@ mapper_registry.map_imperatively(model.Page, model.Page.table, properties=dict( order_by=model.PageAnnotationAssociation.table.c.id, backref="page"), ratings=relation(model.PageRatingAssociation, - order_by=model.PageRatingAssociation.table.c.id, - backref="page"), + order_by=model.PageRatingAssociation.id, + back_populates="page"), average_rating=column_property( - select(func.avg(model.PageRatingAssociation.table.c.rating)).where(model.PageRatingAssociation.table.c.page_id == model.Page.table.c.id).scalar_subquery(), + select(func.avg(model.PageRatingAssociation.rating)).where(model.PageRatingAssociation.page_id == model.Page.table.c.id).scalar_subquery(), deferred=True ), users_shared_with=relation(model.PageUserShareAssociation, back_populates='page') @@ -2372,7 +2365,6 @@ def rating_mapping(rating_class, **kwds): simple_mapping(rating_class, **dict(user=relation(model.User), **kwds)) -rating_mapping(model.PageRatingAssociation) rating_mapping(model.VisualizationRatingAssociation) rating_mapping(model.HistoryDatasetCollectionRatingAssociation) rating_mapping(model.LibraryDatasetCollectionRatingAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index b6746d95829..52e13ea64f5 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -211,6 +211,21 @@ def test_HistoryDatasetAssociationRatingAssociation(model, session, history_data assert stored_obj.rating == rating +def test_PageRatingAssociation(model, session, page, user): + cls = model.PageRatingAssociation + assert cls.__tablename__ == 'page_rating_association' + with dbcleanup(session, cls): + rating = 9 + obj = cls(user, page, rating) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.page == page + assert stored_obj.rating == rating + + def test_PageRevision(model, session, page): cls = model.PageRevision assert cls.__tablename__ == 'page_revision' From af0c8e9217c528944bef6b38397650630304222b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 01:41:32 -0400 Subject: [PATCH 032/225] Map VisualizationRatingAssociation declaratively --- lib/galaxy/model/__init__.py | 11 ++++++++++- lib/galaxy/model/mapping.py | 14 +++----------- test/unit/model/test_model.py | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ba76ae6a974..397c4b71a66 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6957,7 +6957,16 @@ class PageRatingAssociation(ItemRatingAssociation, RepresentById): self.page = page -class VisualizationRatingAssociation(_remove_ItemRatingAssociation, RepresentById): +class VisualizationRatingAssociation(ItemRatingAssociation, RepresentById): + __tablename__ = 'visualization_rating_association' + + id = Column('id', Integer, primary_key=True) + visualization_id = Column('visualization_id', Integer, ForeignKey('visualization.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column('rating', Integer, index=True) + visualization = relationship('Visualization', back_populates='ratings') + user = relationship('User') + def _set_item(self, visualization): self.visualization = visualization diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 9ddbef4888a..c7db322ced0 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1293,13 +1293,6 @@ model.LibraryDatasetCollectionAnnotationAssociation.table = Table( ) # Ratings tables. -model.VisualizationRatingAssociation.table = Table( - "visualization_rating_association", metadata, - Column("id", Integer, primary_key=True), - Column("visualization_id", Integer, ForeignKey("visualization.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("rating", Integer, index=True)) - model.HistoryDatasetCollectionRatingAssociation.table = Table( "history_dataset_collection_rating_association", metadata, Column("id", Integer, primary_key=True), @@ -2302,10 +2295,10 @@ mapper_registry.map_imperatively(model.Visualization, model.Visualization.table, order_by=model.VisualizationAnnotationAssociation.table.c.id, backref="visualization"), ratings=relation(model.VisualizationRatingAssociation, - order_by=model.VisualizationRatingAssociation.table.c.id, - backref="visualization"), + order_by=model.VisualizationRatingAssociation.id, + back_populates="visualization"), average_rating=column_property( - select(func.avg(model.VisualizationRatingAssociation.table.c.rating)).where(model.VisualizationRatingAssociation.table.c.visualization_id == model.Visualization.table.c.id).scalar_subquery(), + select(func.avg(model.VisualizationRatingAssociation.rating)).where(model.VisualizationRatingAssociation.visualization_id == model.Visualization.table.c.id).scalar_subquery(), deferred=True ) )) @@ -2365,7 +2358,6 @@ def rating_mapping(rating_class, **kwds): simple_mapping(rating_class, **dict(user=relation(model.User), **kwds)) -rating_mapping(model.VisualizationRatingAssociation) rating_mapping(model.HistoryDatasetCollectionRatingAssociation) rating_mapping(model.LibraryDatasetCollectionRatingAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 52e13ea64f5..fafed4e7e44 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -471,6 +471,21 @@ def test_UserQuotaAssociation(model, session, user, quota): assert stored_obj.quota == quota +def test_VisualizationRatingAssociation(model, session, visualization, user): + cls = model.VisualizationRatingAssociation + assert cls.__tablename__ == 'visualization_rating_association' + with dbcleanup(session, cls): + rating = 9 + obj = cls(user, visualization, rating) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.visualization == visualization + assert stored_obj.rating == rating + + def test_VisualizationRevision(model, session, visualization): cls = model.VisualizationRevision assert cls.__tablename__ == 'visualization_revision' From d06c8a5515d334ec26d610b59574dafca4aed846 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 12:35:22 -0400 Subject: [PATCH 033/225] Map HistoryDatasetCollectionRatingAssociation declaratively --- lib/galaxy/model/__init__.py | 12 ++++++- lib/galaxy/model/mapping.py | 13 ++----- test/unit/model/test_model.py | 64 +++++++++++++++++++++++------------ 3 files changed, 56 insertions(+), 33 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 397c4b71a66..6fd41fa3438 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6971,7 +6971,17 @@ class VisualizationRatingAssociation(ItemRatingAssociation, RepresentById): self.visualization = visualization -class HistoryDatasetCollectionRatingAssociation(_remove_ItemRatingAssociation, RepresentById): +class HistoryDatasetCollectionRatingAssociation(ItemRatingAssociation, RepresentById): + __tablename__ = 'history_dataset_collection_rating_association' + + id = Column('id', Integer, primary_key=True) + history_dataset_collection_id = Column('history_dataset_collection_id', Integer, + ForeignKey('history_dataset_collection_association.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column('rating', Integer, index=True) + dataset_collection = relationship('HistoryDatasetCollectionAssociation', back_populates='ratings') + user = relationship('User') + def _set_item(self, dataset_collection): self.dataset_collection = dataset_collection diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index c7db322ced0..6a9d2faab69 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1293,14 +1293,6 @@ model.LibraryDatasetCollectionAnnotationAssociation.table = Table( ) # Ratings tables. -model.HistoryDatasetCollectionRatingAssociation.table = Table( - "history_dataset_collection_rating_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_dataset_collection_id", Integer, - ForeignKey("history_dataset_collection_association.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("rating", Integer, index=True)) - model.LibraryDatasetCollectionRatingAssociation.table = Table( "library_dataset_collection_rating_association", metadata, Column("id", Integer, primary_key=True), @@ -1981,8 +1973,8 @@ simple_mapping(model.HistoryDatasetCollectionAssociation, order_by=model.HistoryDatasetCollectionAssociationAnnotationAssociation.table.c.id, backref="history_dataset_collection"), ratings=relation(model.HistoryDatasetCollectionRatingAssociation, - order_by=model.HistoryDatasetCollectionRatingAssociation.table.c.id, - backref="dataset_collection") + order_by=model.HistoryDatasetCollectionRatingAssociation.id, + back_populates="dataset_collection") ) simple_mapping(model.LibraryDatasetCollectionAssociation, @@ -2358,7 +2350,6 @@ def rating_mapping(rating_class, **kwds): simple_mapping(rating_class, **dict(user=relation(model.User), **kwds)) -rating_mapping(model.HistoryDatasetCollectionRatingAssociation) rating_mapping(model.LibraryDatasetCollectionRatingAssociation) mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index fafed4e7e44..69453250665 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -181,21 +181,6 @@ def test_GroupRoleAssociation(model, session, group, role): assert stored_obj.role == role -def test_HistoryRatingAssociation(model, session, history, user): - cls = model.HistoryRatingAssociation - assert cls.__tablename__ == 'history_rating_association' - with dbcleanup(session, cls): - rating = 9 - obj = cls(user, history, rating) - obj_id = persist(session, obj) - - stored_obj = get_stored_obj(session, cls, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.user == user - assert stored_obj.history == history - assert stored_obj.rating == rating - - def test_HistoryDatasetAssociationRatingAssociation(model, session, history_dataset_association, user): cls = model.HistoryDatasetAssociationRatingAssociation assert cls.__tablename__ == 'history_dataset_association_rating_association' @@ -211,6 +196,37 @@ def test_HistoryDatasetAssociationRatingAssociation(model, session, history_data assert stored_obj.rating == rating +def test_HistoryDatasetCollectionRatingAssociation( + model, session, history_dataset_collection_association, user): + cls = model.HistoryDatasetCollectionRatingAssociation + assert cls.__tablename__ == 'history_dataset_collection_rating_association' + with dbcleanup(session, cls): + rating = 9 + obj = cls(user, history_dataset_collection_association, rating) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.dataset_collection == history_dataset_collection_association + assert stored_obj.rating == rating + + +def test_HistoryRatingAssociation(model, session, history, user): + cls = model.HistoryRatingAssociation + assert cls.__tablename__ == 'history_rating_association' + with dbcleanup(session, cls): + rating = 9 + obj = cls(user, history, rating) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.history == history + assert stored_obj.rating == rating + + def test_PageRatingAssociation(model, session, page, user): cls = model.PageRatingAssociation assert cls.__tablename__ == 'page_rating_association' @@ -542,6 +558,18 @@ def cloud_authz(model, session, user, user_authnz_token): yield from dbcleanup_wrapper(session, ca) +@pytest.fixture +def dataset(model, session): + d = model.Dataset() + yield from dbcleanup_wrapper(session, d) + + +@pytest.fixture +def history_dataset_collection_association(model, session): + hdca = model.HistoryDatasetCollectionAssociation() + yield from dbcleanup_wrapper(session, hdca) + + @pytest.fixture def history(model, session): h = model.History() @@ -554,12 +582,6 @@ def history_dataset_association(model, session, dataset): yield from dbcleanup_wrapper(session, hda) -@pytest.fixture -def dataset(model, session): - d = model.Dataset() - yield from dbcleanup_wrapper(session, d) - - @pytest.fixture def galaxy_session(model, session, user): s = model.GalaxySession() From 498b7c4722715abbe56147e28edc8b812c012859 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 12:44:59 -0400 Subject: [PATCH 034/225] Map LibraryDatasetCollectionRatingAssoc declaratively Also, remove temporary code supporting testing for rating models mapped imperatively. --- lib/galaxy/model/__init__.py | 26 +++++++++++--------------- lib/galaxy/model/mapping.py | 22 ++-------------------- test/unit/model/test_model.py | 34 ++++++++++++++++++++++++++++------ 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6fd41fa3438..e7cdd4eea44 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6886,20 +6886,6 @@ class ItemRatingAssociation(Base): raise NotImplementedError() -class _remove_ItemRatingAssociation: - # This is a temporary fix to make the yet-to-be-mapped-declaratively rating - # association models pass the tests. TO BE REMOVED BEFORE REVIEW. - - def __init__(self, user, item, rating=0): - self.user = user - self.rating = rating - self._set_item(item) - - def _set_item(self, item): - """ Set association's item. """ - raise NotImplementedError() - - class HistoryRatingAssociation(ItemRatingAssociation, RepresentById): __tablename__ = 'history_rating_association' @@ -6986,7 +6972,17 @@ class HistoryDatasetCollectionRatingAssociation(ItemRatingAssociation, Represent self.dataset_collection = dataset_collection -class LibraryDatasetCollectionRatingAssociation(_remove_ItemRatingAssociation, RepresentById): +class LibraryDatasetCollectionRatingAssociation(ItemRatingAssociation, RepresentById): + __tablename__ = 'library_dataset_collection_rating_association' + + id = Column('id', Integer, primary_key=True) + library_dataset_collection_id = Column('library_dataset_collection_id', Integer, + ForeignKey('library_dataset_collection_association.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column('rating', Integer, index=True) + dataset_collection = relationship('LibraryDatasetCollectionAssociation', back_populates='ratings') + user = relationship('User') + def _set_item(self, dataset_collection): self.dataset_collection = dataset_collection diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6a9d2faab69..8fa6da64692 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1292,15 +1292,6 @@ model.LibraryDatasetCollectionAnnotationAssociation.table = Table( Column("annotation", TEXT), ) -# Ratings tables. -model.LibraryDatasetCollectionRatingAssociation.table = Table( - "library_dataset_collection_rating_association", metadata, - Column("id", Integer, primary_key=True), - Column("library_dataset_collection_id", Integer, - ForeignKey("library_dataset_collection_association.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("rating", Integer, index=True)) - model.UserPreference.table = Table( "user_preference", metadata, Column("id", Integer, primary_key=True), @@ -1988,8 +1979,8 @@ simple_mapping(model.LibraryDatasetCollectionAssociation, order_by=model.LibraryDatasetCollectionAnnotationAssociation.table.c.id, backref="dataset_collection"), ratings=relation(model.LibraryDatasetCollectionRatingAssociation, - order_by=model.LibraryDatasetCollectionRatingAssociation.table.c.id, - backref="dataset_collection")) + order_by=model.LibraryDatasetCollectionRatingAssociation.id, + back_populates="dataset_collection")) simple_mapping(model.DatasetCollectionElement, hda=relation(model.HistoryDatasetAssociation, @@ -2343,15 +2334,6 @@ annotation_mapping(model.VisualizationAnnotationAssociation) annotation_mapping(model.HistoryDatasetCollectionAssociationAnnotationAssociation) annotation_mapping(model.LibraryDatasetCollectionAnnotationAssociation) - -# Rating tables. -def rating_mapping(rating_class, **kwds): - kwds = {key: relation(value) for key, value in kwds.items()} - simple_mapping(rating_class, **dict(user=relation(model.User), **kwds)) - - -rating_mapping(model.LibraryDatasetCollectionRatingAssociation) - mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( # user=relation( model.User.mapper ), user=relation(model.User), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 69453250665..05bb688f602 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -227,6 +227,22 @@ def test_HistoryRatingAssociation(model, session, history, user): assert stored_obj.rating == rating +def test_LibraryDatasetCollectionRatingAssociation( + model, session, library_dataset_collection_association, user): + cls = model.LibraryDatasetCollectionRatingAssociation + assert cls.__tablename__ == 'library_dataset_collection_rating_association' + with dbcleanup(session, cls): + rating = 9 + obj = cls(user, library_dataset_collection_association, rating) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user == user + assert stored_obj.dataset_collection == library_dataset_collection_association + assert stored_obj.rating == rating + + def test_PageRatingAssociation(model, session, page, user): cls = model.PageRatingAssociation assert cls.__tablename__ == 'page_rating_association' @@ -564,12 +580,6 @@ def dataset(model, session): yield from dbcleanup_wrapper(session, d) -@pytest.fixture -def history_dataset_collection_association(model, session): - hdca = model.HistoryDatasetCollectionAssociation() - yield from dbcleanup_wrapper(session, hdca) - - @pytest.fixture def history(model, session): h = model.History() @@ -582,6 +592,12 @@ def history_dataset_association(model, session, dataset): yield from dbcleanup_wrapper(session, hda) +@pytest.fixture +def history_dataset_collection_association(model, session): + hdca = model.HistoryDatasetCollectionAssociation() + yield from dbcleanup_wrapper(session, hdca) + + @pytest.fixture def galaxy_session(model, session, user): s = model.GalaxySession() @@ -594,6 +610,12 @@ def group(model, session): yield from dbcleanup_wrapper(session, g) +@pytest.fixture +def library_dataset_collection_association(model, session): + ldca = model.LibraryDatasetCollectionAssociation() + yield from dbcleanup_wrapper(session, ldca) + + @pytest.fixture def page(model, session, user): p = model.Page() From 4721920a943c33f94d1842454619d1dfd4c11dd6 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 14:52:35 -0400 Subject: [PATCH 035/225] Map HistoryAnnotationAssociation declaratively --- lib/galaxy/model/__init__.py | 14 ++++++++++++-- lib/galaxy/model/mapping.py | 15 ++------------- test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e7cdd4eea44..b0d824ea07b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6840,8 +6840,18 @@ class ToolTagAssociation(ItemTagAssociation, RepresentById): # Item annotation classes. -class HistoryAnnotationAssociation(RepresentById): - pass +class HistoryAnnotationAssociation(Base, RepresentById): + __tablename__ = 'history_annotation_association' + __table_args__ = ( + Index('ix_history_anno_assoc_annotation', 'annotation', mysql_length=200), + ) + + id = Column('id', Integer, primary_key=True) + history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column('annotation', TEXT) + history = relationship('History', back_populates='annotations') + user = relationship('User') class HistoryDatasetAssociationAnnotationAssociation(RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 8fa6da64692..6264b35fb7b 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1218,16 +1218,6 @@ model.ToolTagAssociation.table = Table( Column("user_value", TrimmedString(255), index=True)) # Annotation tables. - -model.HistoryAnnotationAssociation.table = Table( - "history_annotation_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("annotation", TEXT), - Index('ix_history_anno_assoc_annotation', 'annotation', mysql_length=200), -) - model.HistoryDatasetAssociationAnnotationAssociation.table = Table( "history_dataset_association_annotation_association", metadata, Column("id", Integer, primary_key=True), @@ -1526,8 +1516,8 @@ mapper_registry.map_imperatively(model.History, model.History.table, properties= order_by=model.HistoryTagAssociation.table.c.id, backref="histories"), annotations=relation(model.HistoryAnnotationAssociation, - order_by=model.HistoryAnnotationAssociation.table.c.id, - backref="history"), + order_by=model.HistoryAnnotationAssociation.id, + back_populates="history"), ratings=relation(model.HistoryRatingAssociation, order_by=model.HistoryRatingAssociation.id, back_populates="history"), @@ -2325,7 +2315,6 @@ def annotation_mapping(annotation_class, **kwds): simple_mapping(annotation_class, **dict(user=relation(model.User), **kwds)) -annotation_mapping(model.HistoryAnnotationAssociation) annotation_mapping(model.HistoryDatasetAssociationAnnotationAssociation) annotation_mapping(model.StoredWorkflowAnnotationAssociation) annotation_mapping(model.WorkflowStepAnnotationAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 05bb688f602..272160be90d 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -181,6 +181,25 @@ def test_GroupRoleAssociation(model, session, group, role): assert stored_obj.role == role +def test_HistoryAnnotationAssociation(model, session, history, user): + cls = model.HistoryAnnotationAssociation + assert cls.__tablename__ == 'history_annotation_association' + assert has_index(cls.__table__, ('annotation',)) + with dbcleanup(session, cls): + annotation = 'a' + obj = cls() + obj.user = user + obj.history = history + obj.annotation = annotation + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history == history + assert stored_obj.user == user + assert stored_obj.annotation == annotation + + def test_HistoryDatasetAssociationRatingAssociation(model, session, history_dataset_association, user): cls = model.HistoryDatasetAssociationRatingAssociation assert cls.__tablename__ == 'history_dataset_association_rating_association' From a30e6950db3031aa644de727ccc58560d8fc2cce Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 15:43:22 -0400 Subject: [PATCH 036/225] Map HDAAnnotationAssociation declaratively --- lib/galaxy/model/__init__.py | 15 +++++++++++++-- lib/galaxy/model/mapping.py | 15 ++------------- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b0d824ea07b..9c02575cbb6 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6854,8 +6854,19 @@ class HistoryAnnotationAssociation(Base, RepresentById): user = relationship('User') -class HistoryDatasetAssociationAnnotationAssociation(RepresentById): - pass +class HistoryDatasetAssociationAnnotationAssociation(Base, RepresentById): + __tablename__ = 'history_dataset_association_annotation_association' + __table_args__ = ( + Index('ix_history_dataset_anno_assoc_annotation', 'annotation', mysql_length=200), + ) + + id = Column('id', Integer, primary_key=True) + history_dataset_association_id = Column('history_dataset_association_id', Integer, + ForeignKey('history_dataset_association.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column('annotation', TEXT) + hda = relationship('HistoryDatasetAssociation', back_populates='annotations') + user = relationship('User') class StoredWorkflowAnnotationAssociation(RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6264b35fb7b..d4b4dbabe56 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1218,16 +1218,6 @@ model.ToolTagAssociation.table = Table( Column("user_value", TrimmedString(255), index=True)) # Annotation tables. -model.HistoryDatasetAssociationAnnotationAssociation.table = Table( - "history_dataset_association_annotation_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_dataset_association_id", Integer, - ForeignKey("history_dataset_association.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("annotation", TEXT), - Index('ix_history_dataset_anno_assoc_annotation', 'annotation', mysql_length=200), -) - model.StoredWorkflowAnnotationAssociation.table = Table( "stored_workflow_annotation_association", metadata, Column("id", Integer, primary_key=True), @@ -1394,8 +1384,8 @@ simple_mapping(model.HistoryDatasetAssociation, order_by=model.HistoryDatasetAssociationTagAssociation.table.c.id, backref='history_tag_associations'), annotations=relation(model.HistoryDatasetAssociationAnnotationAssociation, - order_by=model.HistoryDatasetAssociationAnnotationAssociation.table.c.id, - backref="hda"), + order_by=model.HistoryDatasetAssociationAnnotationAssociation.id, + back_populates="hda"), ratings=relation(model.HistoryDatasetAssociationRatingAssociation, order_by=model.HistoryDatasetAssociationRatingAssociation.id, back_populates="history_dataset_association"), @@ -2315,7 +2305,6 @@ def annotation_mapping(annotation_class, **kwds): simple_mapping(annotation_class, **dict(user=relation(model.User), **kwds)) -annotation_mapping(model.HistoryDatasetAssociationAnnotationAssociation) annotation_mapping(model.StoredWorkflowAnnotationAssociation) annotation_mapping(model.WorkflowStepAnnotationAssociation) annotation_mapping(model.PageAnnotationAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 272160be90d..2e20999eaea 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -200,6 +200,26 @@ def test_HistoryAnnotationAssociation(model, session, history, user): assert stored_obj.annotation == annotation +def test_HistoryDatasetAssociationAnnotationAssociation( + model, session, history_dataset_association, user): + cls = model.HistoryDatasetAssociationAnnotationAssociation + assert cls.__tablename__ == 'history_dataset_association_annotation_association' + assert has_index(cls.__table__, ('annotation',)) + with dbcleanup(session, cls): + annotation = 'a' + obj = cls() + obj.user = user + obj.hda = history_dataset_association + obj.annotation = annotation + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.hda == history_dataset_association + assert stored_obj.user == user + assert stored_obj.annotation == annotation + + def test_HistoryDatasetAssociationRatingAssociation(model, session, history_dataset_association, user): cls = model.HistoryDatasetAssociationRatingAssociation assert cls.__tablename__ == 'history_dataset_association_rating_association' From 400e01eefbde60e877496065231a0707160dd20f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 17:34:22 -0400 Subject: [PATCH 037/225] Map StoredWorkflowAnnotationAssociation declaratively --- lib/galaxy/model/__init__.py | 15 +++++++++++++-- lib/galaxy/model/mapping.py | 14 ++------------ test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9c02575cbb6..45d49895110 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6869,8 +6869,19 @@ class HistoryDatasetAssociationAnnotationAssociation(Base, RepresentById): user = relationship('User') -class StoredWorkflowAnnotationAssociation(RepresentById): - pass +class StoredWorkflowAnnotationAssociation(Base, RepresentById): + __tablename__ = 'stored_workflow_annotation_association' + __table_args__ = ( + Index('ix_stored_workflow_ann_assoc_annotation', 'annotation', mysql_length=200), + ) + + id = Column('id', Integer, primary_key=True) + stored_workflow_id = Column('stored_workflow_id', Integer, + ForeignKey('stored_workflow.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column('annotation', TEXT) + stored_workflow = relationship('StoredWorkflow', back_populates='annotations') + user = relationship('User') class WorkflowStepAnnotationAssociation(RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index d4b4dbabe56..543b6baaa88 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1218,15 +1218,6 @@ model.ToolTagAssociation.table = Table( Column("user_value", TrimmedString(255), index=True)) # Annotation tables. -model.StoredWorkflowAnnotationAssociation.table = Table( - "stored_workflow_annotation_association", metadata, - Column("id", Integer, primary_key=True), - Column("stored_workflow_id", Integer, ForeignKey("stored_workflow.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("annotation", TEXT), - Index('ix_stored_workflow_ann_assoc_annotation', 'annotation', mysql_length=200), -) - model.WorkflowStepAnnotationAssociation.table = Table( "workflow_step_annotation_association", metadata, Column("id", Integer, primary_key=True), @@ -2068,8 +2059,8 @@ mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.tabl viewonly=True, order_by=model.StoredWorkflowTagAssociation.table.c.id), annotations=relation(model.StoredWorkflowAnnotationAssociation, - order_by=model.StoredWorkflowAnnotationAssociation.table.c.id, - backref="stored_workflow"), + order_by=model.StoredWorkflowAnnotationAssociation.id, + back_populates="stored_workflow"), ratings=relation(model.StoredWorkflowRatingAssociation, order_by=model.StoredWorkflowRatingAssociation.id, back_populates="stored_workflow"), @@ -2305,7 +2296,6 @@ def annotation_mapping(annotation_class, **kwds): simple_mapping(annotation_class, **dict(user=relation(model.User), **kwds)) -annotation_mapping(model.StoredWorkflowAnnotationAssociation) annotation_mapping(model.WorkflowStepAnnotationAssociation) annotation_mapping(model.PageAnnotationAssociation) annotation_mapping(model.VisualizationAnnotationAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 2e20999eaea..7735cdf6051 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -432,6 +432,25 @@ def test_Role(model, session): assert stored_obj.deleted is False +def test_StoredWorkflowAnnotationAssociation(model, session, stored_workflow, user): + cls = model.StoredWorkflowAnnotationAssociation + assert cls.__tablename__ == 'stored_workflow_annotation_association' + assert has_index(cls.__table__, ('annotation',)) + with dbcleanup(session, cls): + annotation = 'a' + obj = cls() + obj.user = user + obj.stored_workflow = stored_workflow + obj.annotation = annotation + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.stored_workflow == stored_workflow + assert stored_obj.user == user + assert stored_obj.annotation == annotation + + def test_StoredWorkflowRatingAssociation(model, session, stored_workflow, user): cls = model.StoredWorkflowRatingAssociation assert cls.__tablename__ == 'stored_workflow_rating_association' From 5c8df1b03c20a0e94bf0de8ea0f6433eafd3f978 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 17:40:47 -0400 Subject: [PATCH 038/225] Map WorkflowStepAnnotationAssociation declaratively --- lib/galaxy/model/__init__.py | 15 +++++++++++++-- lib/galaxy/model/mapping.py | 14 ++------------ test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 45d49895110..81e27f60018 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6884,8 +6884,19 @@ class StoredWorkflowAnnotationAssociation(Base, RepresentById): user = relationship('User') -class WorkflowStepAnnotationAssociation(RepresentById): - pass +class WorkflowStepAnnotationAssociation(Base, RepresentById): + __tablename__ = 'workflow_step_annotation_association' + __table_args__ = ( + Index('ix_workflow_step_ann_assoc_annotation', 'annotation', mysql_length=200), + ) + + id = Column('id', Integer, primary_key=True) + workflow_step_id = Column('workflow_step_id', Integer, + ForeignKey('workflow_step.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column('annotation', TEXT) + workflow_step = relationship('WorkflowStep', back_populates='annotations') + user = relationship('User') class PageAnnotationAssociation(RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 543b6baaa88..6457f06887e 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1218,15 +1218,6 @@ model.ToolTagAssociation.table = Table( Column("user_value", TrimmedString(255), index=True)) # Annotation tables. -model.WorkflowStepAnnotationAssociation.table = Table( - "workflow_step_annotation_association", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("annotation", TEXT), - Index('ix_workflow_step_ann_assoc_annotation', 'annotation', mysql_length=200), -) - model.PageAnnotationAssociation.table = Table( "page_annotation_association", metadata, Column("id", Integer, primary_key=True), @@ -2002,8 +1993,8 @@ mapper_registry.map_imperatively(model.WorkflowStep, model.WorkflowStep.table, p order_by=model.WorkflowStepTagAssociation.table.c.id, backref="workflow_steps"), annotations=relation(model.WorkflowStepAnnotationAssociation, - order_by=model.WorkflowStepAnnotationAssociation.table.c.id, - backref="workflow_step") + order_by=model.WorkflowStepAnnotationAssociation.id, + back_populates="workflow_step") )) mapper_registry.map_imperatively(model.WorkflowStepInput, model.WorkflowStepInput.table, properties=dict( @@ -2296,7 +2287,6 @@ def annotation_mapping(annotation_class, **kwds): simple_mapping(annotation_class, **dict(user=relation(model.User), **kwds)) -annotation_mapping(model.WorkflowStepAnnotationAssociation) annotation_mapping(model.PageAnnotationAssociation) annotation_mapping(model.VisualizationAnnotationAssociation) annotation_mapping(model.HistoryDatasetCollectionAssociationAnnotationAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 7735cdf6051..4bcb34e46c9 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -613,6 +613,25 @@ def test_WorkerProcess(model, session): assert stored_obj.update_time +def test_WorkflowStepAnnotationAssociation(model, session, workflow_step, user): + cls = model.WorkflowStepAnnotationAssociation + assert cls.__tablename__ == 'workflow_step_annotation_association' + assert has_index(cls.__table__, ('annotation',)) + with dbcleanup(session, cls): + annotation = 'a' + obj = cls() + obj.user = user + obj.workflow_step = workflow_step + obj.annotation = annotation + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_step == workflow_step + assert stored_obj.user == user + assert stored_obj.annotation == annotation + + @pytest.fixture(scope='module') def model(): db_uri = 'sqlite:///:memory:' From 02eec6e25bf066f3e66e96891bcdf2559dff4262 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 22:59:34 -0400 Subject: [PATCH 039/225] Map PageAnnotationAssociation decaratively --- lib/galaxy/model/__init__.py | 14 ++++++++++++-- lib/galaxy/model/mapping.py | 14 ++------------ test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 81e27f60018..430b25f9d97 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6899,8 +6899,18 @@ class WorkflowStepAnnotationAssociation(Base, RepresentById): user = relationship('User') -class PageAnnotationAssociation(RepresentById): - pass +class PageAnnotationAssociation(Base, RepresentById): + __tablename__ = 'page_annotation_association' + __table_args__ = ( + Index('ix_page_annotation_association_annotation', 'annotation', mysql_length=200), + ) + + id = Column('id', Integer, primary_key=True) + page_id = Column('page_id', Integer, ForeignKey('page.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column('annotation', TEXT) + page = relationship('Page', back_populates='annotations') + user = relationship('User') class VisualizationAnnotationAssociation(RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6457f06887e..1485a2d94df 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1218,15 +1218,6 @@ model.ToolTagAssociation.table = Table( Column("user_value", TrimmedString(255), index=True)) # Annotation tables. -model.PageAnnotationAssociation.table = Table( - "page_annotation_association", metadata, - Column("id", Integer, primary_key=True), - Column("page_id", Integer, ForeignKey("page.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("annotation", TEXT), - Index('ix_page_annotation_association_annotation', 'annotation', mysql_length=200), -) - model.VisualizationAnnotationAssociation.table = Table( "visualization_annotation_association", metadata, Column("id", Integer, primary_key=True), @@ -2206,8 +2197,8 @@ mapper_registry.map_imperatively(model.Page, model.Page.table, properties=dict( order_by=model.PageTagAssociation.table.c.id, backref="pages"), annotations=relation(model.PageAnnotationAssociation, - order_by=model.PageAnnotationAssociation.table.c.id, - backref="page"), + order_by=model.PageAnnotationAssociation.id, + back_populates="page"), ratings=relation(model.PageRatingAssociation, order_by=model.PageRatingAssociation.id, back_populates="page"), @@ -2287,7 +2278,6 @@ def annotation_mapping(annotation_class, **kwds): simple_mapping(annotation_class, **dict(user=relation(model.User), **kwds)) -annotation_mapping(model.PageAnnotationAssociation) annotation_mapping(model.VisualizationAnnotationAssociation) annotation_mapping(model.HistoryDatasetCollectionAssociationAnnotationAssociation) annotation_mapping(model.LibraryDatasetCollectionAnnotationAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 4bcb34e46c9..921e767fb02 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -282,6 +282,25 @@ def test_LibraryDatasetCollectionRatingAssociation( assert stored_obj.rating == rating +def test_PageAnnotationAssociation(model, session, page, user): + cls = model.PageAnnotationAssociation + assert cls.__tablename__ == 'page_annotation_association' + assert has_index(cls.__table__, ('annotation',)) + with dbcleanup(session, cls): + annotation = 'a' + obj = cls() + obj.user = user + obj.page = page + obj.annotation = annotation + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.page == page + assert stored_obj.user == user + assert stored_obj.annotation == annotation + + def test_PageRatingAssociation(model, session, page, user): cls = model.PageRatingAssociation assert cls.__tablename__ == 'page_rating_association' From e988c8b41d5b7789e931a87f1cfe3e092c410ff6 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 23:04:26 -0400 Subject: [PATCH 040/225] Map VisualizationAnnotationAssociation declaratively --- lib/galaxy/model/__init__.py | 14 ++++++++++++-- lib/galaxy/model/mapping.py | 14 ++------------ test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 430b25f9d97..9e1f4327864 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6913,8 +6913,18 @@ class PageAnnotationAssociation(Base, RepresentById): user = relationship('User') -class VisualizationAnnotationAssociation(RepresentById): - pass +class VisualizationAnnotationAssociation(Base, RepresentById): + __tablename__ = 'visualization_annotation_association' + __table_args__ = ( + Index('ix_visualization_annotation_association_annotation', 'annotation', mysql_length=200), + ) + + id = Column('id', Integer, primary_key=True) + visualization_id = Column('visualization_id', Integer, ForeignKey('visualization.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column('annotation', TEXT) + visualization = relationship('Visualization', back_populates='annotations') + user = relationship('User') class HistoryDatasetCollectionAssociationAnnotationAssociation(RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 1485a2d94df..b3d16ece164 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1218,15 +1218,6 @@ model.ToolTagAssociation.table = Table( Column("user_value", TrimmedString(255), index=True)) # Annotation tables. -model.VisualizationAnnotationAssociation.table = Table( - "visualization_annotation_association", metadata, - Column("id", Integer, primary_key=True), - Column("visualization_id", Integer, ForeignKey("visualization.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("annotation", TEXT), - Index('ix_visualization_annotation_association_annotation', 'annotation', mysql_length=200), -) - model.HistoryDatasetCollectionAssociationAnnotationAssociation.table = Table( "history_dataset_collection_annotation_association", metadata, Column("id", Integer, primary_key=True), @@ -2228,8 +2219,8 @@ mapper_registry.map_imperatively(model.Visualization, model.Visualization.table, order_by=model.VisualizationTagAssociation.table.c.id, backref="visualizations"), annotations=relation(model.VisualizationAnnotationAssociation, - order_by=model.VisualizationAnnotationAssociation.table.c.id, - backref="visualization"), + order_by=model.VisualizationAnnotationAssociation.id, + back_populates="visualization"), ratings=relation(model.VisualizationRatingAssociation, order_by=model.VisualizationRatingAssociation.id, back_populates="visualization"), @@ -2278,7 +2269,6 @@ def annotation_mapping(annotation_class, **kwds): simple_mapping(annotation_class, **dict(user=relation(model.User), **kwds)) -annotation_mapping(model.VisualizationAnnotationAssociation) annotation_mapping(model.HistoryDatasetCollectionAssociationAnnotationAssociation) annotation_mapping(model.LibraryDatasetCollectionAnnotationAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 921e767fb02..ae6f69409a5 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -580,6 +580,25 @@ def test_UserQuotaAssociation(model, session, user, quota): assert stored_obj.quota == quota +def test_VisualizationAnnotationAssociation(model, session, visualization, user): + cls = model.VisualizationAnnotationAssociation + assert cls.__tablename__ == 'visualization_annotation_association' + assert has_index(cls.__table__, ('annotation',)) + with dbcleanup(session, cls): + annotation = 'a' + obj = cls() + obj.user = user + obj.visualization = visualization + obj.annotation = annotation + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.visualization == visualization + assert stored_obj.user == user + assert stored_obj.annotation == annotation + + def test_VisualizationRatingAssociation(model, session, visualization, user): cls = model.VisualizationRatingAssociation assert cls.__tablename__ == 'visualization_rating_association' From 5e679441e2d4d1fe296d48cd0f0d2e41bb9bd990 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 23:18:21 -0400 Subject: [PATCH 041/225] Map HDCAAnnotationAssociation declaratively --- lib/galaxy/model/__init__.py | 13 +++++++++++-- lib/galaxy/model/mapping.py | 14 ++------------ test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9e1f4327864..d0570baad98 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6927,8 +6927,17 @@ class VisualizationAnnotationAssociation(Base, RepresentById): user = relationship('User') -class HistoryDatasetCollectionAssociationAnnotationAssociation(RepresentById): - pass +class HistoryDatasetCollectionAssociationAnnotationAssociation(Base, RepresentById): + __tablename__ = 'history_dataset_collection_annotation_association' + + id = Column('id', Integer, primary_key=True) + history_dataset_collection_id = Column('history_dataset_collection_id', Integer, + ForeignKey('history_dataset_collection_association.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column('annotation', TEXT) + history_dataset_collection = relationship('HistoryDatasetCollectionAssociation', + back_populates='annotations') + user = relationship('User') class LibraryDatasetCollectionAnnotationAssociation(RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index b3d16ece164..9c98ef8b6d0 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1218,15 +1218,6 @@ model.ToolTagAssociation.table = Table( Column("user_value", TrimmedString(255), index=True)) # Annotation tables. -model.HistoryDatasetCollectionAssociationAnnotationAssociation.table = Table( - "history_dataset_collection_annotation_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_dataset_collection_id", Integer, - ForeignKey("history_dataset_collection_association.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("annotation", TEXT), -) - model.LibraryDatasetCollectionAnnotationAssociation.table = Table( "library_dataset_collection_annotation_association", metadata, Column("id", Integer, primary_key=True), @@ -1905,8 +1896,8 @@ simple_mapping(model.HistoryDatasetCollectionAssociation, order_by=model.HistoryDatasetCollectionTagAssociation.table.c.id, backref='dataset_collections'), annotations=relation(model.HistoryDatasetCollectionAssociationAnnotationAssociation, - order_by=model.HistoryDatasetCollectionAssociationAnnotationAssociation.table.c.id, - backref="history_dataset_collection"), + order_by=model.HistoryDatasetCollectionAssociationAnnotationAssociation.id, + back_populates="history_dataset_collection"), ratings=relation(model.HistoryDatasetCollectionRatingAssociation, order_by=model.HistoryDatasetCollectionRatingAssociation.id, back_populates="dataset_collection") @@ -2269,7 +2260,6 @@ def annotation_mapping(annotation_class, **kwds): simple_mapping(annotation_class, **dict(user=relation(model.User), **kwds)) -annotation_mapping(model.HistoryDatasetCollectionAssociationAnnotationAssociation) annotation_mapping(model.LibraryDatasetCollectionAnnotationAssociation) mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index ae6f69409a5..c1c763d74ff 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -235,6 +235,25 @@ def test_HistoryDatasetAssociationRatingAssociation(model, session, history_data assert stored_obj.rating == rating +def test_HistoryDatasetCollectionAssociationAnnotationAssociation( + model, session, history_dataset_collection_association, user): + cls = model.HistoryDatasetCollectionAssociationAnnotationAssociation + assert cls.__tablename__ == 'history_dataset_collection_annotation_association' + with dbcleanup(session, cls): + annotation = 'a' + obj = cls() + obj.user = user + obj.history_dataset_collection = history_dataset_collection_association + obj.annotation = annotation + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_collection == history_dataset_collection_association + assert stored_obj.user == user + assert stored_obj.annotation == annotation + + def test_HistoryDatasetCollectionRatingAssociation( model, session, history_dataset_collection_association, user): cls = model.HistoryDatasetCollectionRatingAssociation From 746d4dd8e828ad4479ed23dd2208f0644ff5ee88 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 16 Jun 2021 23:27:51 -0400 Subject: [PATCH 042/225] Map LDCAnnotationAssociation declaratively --- lib/galaxy/model/__init__.py | 13 +++++++++++-- lib/galaxy/model/mapping.py | 23 ++--------------------- test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index d0570baad98..4a82841bbb0 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6940,8 +6940,17 @@ class HistoryDatasetCollectionAssociationAnnotationAssociation(Base, RepresentBy user = relationship('User') -class LibraryDatasetCollectionAnnotationAssociation(RepresentById): - pass +class LibraryDatasetCollectionAnnotationAssociation(Base, RepresentById): + __tablename__ = 'library_dataset_collection_annotation_association' + + id = Column('id', Integer, primary_key=True) + library_dataset_collection_id = Column('library_dataset_collection_id', Integer, + ForeignKey('library_dataset_collection_association.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column('annotation', TEXT) + dataset_collection = relationship('LibraryDatasetCollectionAssociation', + back_populates='annotations') + user = relationship('User') # Item rating classes. diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 9c98ef8b6d0..a7331d7c90b 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1217,16 +1217,6 @@ model.ToolTagAssociation.table = Table( Column("value", TrimmedString(255), index=True), Column("user_value", TrimmedString(255), index=True)) -# Annotation tables. -model.LibraryDatasetCollectionAnnotationAssociation.table = Table( - "library_dataset_collection_annotation_association", metadata, - Column("id", Integer, primary_key=True), - Column("library_dataset_collection_id", Integer, - ForeignKey("library_dataset_collection_association.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("annotation", TEXT), -) - model.UserPreference.table = Table( "user_preference", metadata, Column("id", Integer, primary_key=True), @@ -1911,8 +1901,8 @@ simple_mapping(model.LibraryDatasetCollectionAssociation, order_by=model.LibraryDatasetCollectionTagAssociation.table.c.id, backref='dataset_collections'), annotations=relation(model.LibraryDatasetCollectionAnnotationAssociation, - order_by=model.LibraryDatasetCollectionAnnotationAssociation.table.c.id, - backref="dataset_collection"), + order_by=model.LibraryDatasetCollectionAnnotationAssociation.id, + back_populates="dataset_collection"), ratings=relation(model.LibraryDatasetCollectionRatingAssociation, order_by=model.LibraryDatasetCollectionRatingAssociation.id, back_populates="dataset_collection")) @@ -2253,15 +2243,6 @@ tag_mapping(model.HistoryDatasetCollectionTagAssociation, "tagged_history_datase tag_mapping(model.LibraryDatasetCollectionTagAssociation, "tagged_library_dataset_collections") tag_mapping(model.ToolTagAssociation, "tagged_tools") - -# Annotation tables. -def annotation_mapping(annotation_class, **kwds): - kwds = {key: relation(value) for key, value in kwds.items()} - simple_mapping(annotation_class, **dict(user=relation(model.User), **kwds)) - - -annotation_mapping(model.LibraryDatasetCollectionAnnotationAssociation) - mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( # user=relation( model.User.mapper ), user=relation(model.User), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index c1c763d74ff..75537f87600 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -285,6 +285,25 @@ def test_HistoryRatingAssociation(model, session, history, user): assert stored_obj.rating == rating +def test_LibraryDatasetCollectionAnnotationAssociation( + model, session, library_dataset_collection_association, user): + cls = model.LibraryDatasetCollectionAnnotationAssociation + assert cls.__tablename__ == 'library_dataset_collection_annotation_association' + with dbcleanup(session, cls): + annotation = 'a' + obj = cls() + obj.user = user + obj.dataset_collection = library_dataset_collection_association + obj.annotation = annotation + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.dataset_collection == library_dataset_collection_association + assert stored_obj.user == user + assert stored_obj.annotation == annotation + + def test_LibraryDatasetCollectionRatingAssociation( model, session, library_dataset_collection_association, user): cls = model.LibraryDatasetCollectionRatingAssociation From 9b52409851ce6bd410503938223a0a83b7d8d800 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 23 Jun 2021 12:15:41 -0400 Subject: [PATCH 043/225] Refactor, fix bug in tag assoc models (see note) 1. Do not pass id: assigned by db 2. Do not override value of self.value in constructor: passed as argument 3. Add test for HistoryTagAssociation (exposes bug^) --- lib/galaxy/model/__init__.py | 5 ++--- test/unit/model/test_model.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4a82841bbb0..2d1db2ab051 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6771,13 +6771,12 @@ class ItemTagAssociation(Dictifiable): super().__init_subclass__(**kwargs) cls.associated_item_names.append(cls.__name__.replace("TagAssociation", "")) - def __init__(self, id=None, user=None, item_id=None, tag_id=None, user_tname=None, value=None): - self.id = id + def __init__(self, user=None, item_id=None, tag_id=None, user_tname=None, value=None): self.user = user self.item_id = item_id self.tag_id = tag_id self.user_tname = user_tname - self.value = None + self.value = value self.user_value = None def copy(self, cls=None): diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 75537f87600..e384844048e 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -285,6 +285,25 @@ def test_HistoryRatingAssociation(model, session, history, user): assert stored_obj.rating == rating +def test_HistoryTagAssociation(model, session, history, tag, user): + cls = model.HistoryTagAssociation + with dbcleanup(session, cls): + user_tname, value, user_value = 'a', 'b', 'c' + obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.history_id = history.id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_id == history.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user == user + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_LibraryDatasetCollectionAnnotationAssociation( model, session, library_dataset_collection_association, user): cls = model.LibraryDatasetCollectionAnnotationAssociation @@ -795,6 +814,12 @@ def stored_workflow(model, session, user): yield from dbcleanup_wrapper(session, w) +@pytest.fixture +def tag(model, session): + t = model.Tag() + yield from dbcleanup_wrapper(session, t) + + @pytest.fixture def user(model, session): u = model.User(email='test@example.com', password='password') From 36076be67589080cc21c0d68c85e263c743563f8 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 23 Jun 2021 17:42:27 -0400 Subject: [PATCH 044/225] Add unit test for HDATagAssociation --- test/unit/model/test_model.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index e384844048e..36cc760e679 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -235,6 +235,27 @@ def test_HistoryDatasetAssociationRatingAssociation(model, session, history_data assert stored_obj.rating == rating +def test_HistoryDatasetAssociationTagAssociation( + model, session, history_dataset_association, tag, user): + cls = model.HistoryDatasetAssociationTagAssociation + # TODO assert cls.__tablename__ == '' + with dbcleanup(session, cls): + user_tname, value, user_value = 'a', 'b', 'c' + obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.history_dataset_association_id = history_dataset_association.id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user == user + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_HistoryDatasetCollectionAssociationAnnotationAssociation( model, session, history_dataset_collection_association, user): cls = model.HistoryDatasetCollectionAssociationAnnotationAssociation @@ -287,6 +308,7 @@ def test_HistoryRatingAssociation(model, session, history, user): def test_HistoryTagAssociation(model, session, history, tag, user): cls = model.HistoryTagAssociation + # TODO assert cls.__tablename__ == '' with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) From ee59bacdb056d88c371e8760c3eeb2a89d88f8ac Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 23 Jun 2021 17:49:22 -0400 Subject: [PATCH 045/225] Add unit test for LDDATagAssociation --- test/unit/model/test_model.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 36cc760e679..dd8c8ab4ad9 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -361,6 +361,27 @@ def test_LibraryDatasetCollectionRatingAssociation( assert stored_obj.rating == rating +def test_LibraryDatasetDatasetAssociationTagAssociation( + model, session, library_dataset_dataset_association, tag, user): + cls = model.LibraryDatasetDatasetAssociationTagAssociation + # TODO assert cls.__tablename__ == '' + with dbcleanup(session, cls): + user_tname, value, user_value = 'a', 'b', 'c' + obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.library_dataset_dataset_association_id = library_dataset_dataset_association.id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user == user + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_PageAnnotationAssociation(model, session, page, user): cls = model.PageAnnotationAssociation assert cls.__tablename__ == 'page_annotation_association' @@ -810,6 +831,12 @@ def library_dataset_collection_association(model, session): yield from dbcleanup_wrapper(session, ldca) +@pytest.fixture +def library_dataset_dataset_association(model, session): + ldda = model.LibraryDatasetDatasetAssociation() + yield from dbcleanup_wrapper(session, ldda) + + @pytest.fixture def page(model, session, user): p = model.Page() From e252e0f9b54801c6fba90e272efa28c06f79b6e2 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 23 Jun 2021 17:51:35 -0400 Subject: [PATCH 046/225] Add unit test for PageTagAssociation --- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index dd8c8ab4ad9..aeeb5ec2493 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -435,6 +435,26 @@ def test_PageRevision(model, session, page): assert stored_obj.page.id == page.id +def test_PageTagAssociation(model, session, page, tag, user): + cls = model.PageTagAssociation + # TODO assert cls.__tablename__ == '' + with dbcleanup(session, cls): + user_tname, value, user_value = 'a', 'b', 'c' + obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.page_id = page.id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.page_id == page.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user == user + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_PageUserShareAssociation(model, session, page, user): cls = model.PageUserShareAssociation assert cls.__tablename__ == 'page_user_share_association' From fa9d779793703217ae9ed8af7eaeaea2fc822538 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 23 Jun 2021 17:54:32 -0400 Subject: [PATCH 047/225] Add unit test for StoredWorkflowTagAssociation --- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index aeeb5ec2493..7e862165b0f 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -605,6 +605,26 @@ def test_StoredWorkflowRatingAssociation(model, session, stored_workflow, user): assert stored_obj.rating == rating +def test_StoredWorkflowTagAssociation(model, session, stored_workflow, tag, user): + cls = model.StoredWorkflowTagAssociation + # TODO assert cls.__tablename__ == '' + with dbcleanup(session, cls): + user_tname, value, user_value = 'a', 'b', 'c' + obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.stored_workflow_id = stored_workflow.id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.stored_workflow_id == stored_workflow.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user == user + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def testUserAction(model, session, user, galaxy_session): cls = model.UserAction assert cls.__tablename__ == 'user_action' From c832708c102ecfc46804baf7d20908a7aa63d91e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 23 Jun 2021 17:56:34 -0400 Subject: [PATCH 048/225] Add unit test for WorkflowStepTagAssociation --- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 7e862165b0f..0d7a12c8317 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -810,6 +810,26 @@ def test_WorkflowStepAnnotationAssociation(model, session, workflow_step, user): assert stored_obj.annotation == annotation +def test_WorkflowStepTagAssociation(model, session, workflow_step, tag, user): + cls = model.WorkflowStepTagAssociation + # TODO assert cls.__tablename__ == '' + with dbcleanup(session, cls): + user_tname, value, user_value = 'a', 'b', 'c' + obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.workflow_step_id = workflow_step.id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user == user + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + @pytest.fixture(scope='module') def model(): db_uri = 'sqlite:///:memory:' From 73a50cf64c2a6af13d91ada2a2d7b98f7111769e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 23 Jun 2021 17:59:41 -0400 Subject: [PATCH 049/225] Add unit test for VisualizationTagAssociaion --- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 0d7a12c8317..ec6d1842f0f 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -830,6 +830,26 @@ def test_WorkflowStepTagAssociation(model, session, workflow_step, tag, user): assert stored_obj.user_value == user_value +def test_VisualizationTagAssociation(model, session, visualization, tag, user): + cls = model.VisualizationTagAssociation + # TODO assert cls.__tablename__ == '' + with dbcleanup(session, cls): + user_tname, value, user_value = 'a', 'b', 'c' + obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.visualization_id = visualization.id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.visualization_id == visualization.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user == user + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + @pytest.fixture(scope='module') def model(): db_uri = 'sqlite:///:memory:' From af57256070530a1a92c9d453f1ad0e94a856abca Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 23 Jun 2021 18:19:46 -0400 Subject: [PATCH 050/225] Add unit test for HDCTagAssociation --- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index ec6d1842f0f..05fa8b480eb 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -291,6 +291,26 @@ def test_HistoryDatasetCollectionRatingAssociation( assert stored_obj.rating == rating +def test_HistoryDatasetCollectionTagAssociation(model, session, history_dataset_collection_association, tag, user): + cls = model.HistoryDatasetCollectionTagAssociation + # TODO assert cls.__tablename__ == '' + with dbcleanup(session, cls): + user_tname, value, user_value = 'a', 'b', 'c' + obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.history_dataset_collection_id = history_dataset_collection_association.id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user == user + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_HistoryRatingAssociation(model, session, history, user): cls = model.HistoryRatingAssociation assert cls.__tablename__ == 'history_rating_association' From a53f26578cb5940aa5104757a012d67fccb9fea4 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 23 Jun 2021 18:21:48 -0400 Subject: [PATCH 051/225] Add unit test for LDCTagAssociation --- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 05fa8b480eb..67db80ba30d 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -365,6 +365,26 @@ def test_LibraryDatasetCollectionAnnotationAssociation( assert stored_obj.annotation == annotation +def test_LibraryDatasetCollectionTagAssociation(model, session, library_dataset_collection_association, tag, user): + cls = model.LibraryDatasetCollectionTagAssociation + # TODO assert cls.__tablename__ == '' + with dbcleanup(session, cls): + user_tname, value, user_value = 'a', 'b', 'c' + obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.library_dataset_collection_id = library_dataset_collection_association.id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user == user + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_LibraryDatasetCollectionRatingAssociation( model, session, library_dataset_collection_association, user): cls = model.LibraryDatasetCollectionRatingAssociation From a7282eb3e59f9fdf9282f31db219138ca64e5899 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 23 Jun 2021 18:32:32 -0400 Subject: [PATCH 052/225] Add unit test for ToolTagAssociation --- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 67db80ba30d..11952c9001f 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -665,6 +665,26 @@ def test_StoredWorkflowTagAssociation(model, session, stored_workflow, tag, user assert stored_obj.user_value == user_value +def test_ToolTagAssociation(model, session, tag, user): + cls = model.ToolTagAssociation + # TODO assert cls.__tablename__ == '' + with dbcleanup(session, cls): + user_tname, value, user_value, tool_id = 'a', 'b', 'c', 'd' + obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.tool_id = tool_id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.tool_id == tool_id + assert stored_obj.tag_id == tag.id + assert stored_obj.user == user + assert stored_obj.user_tname == user_tname + assert stored_obj.user_value == user_value + assert stored_obj.value is None # TODO this is a confusing special case; improve? + + def testUserAction(model, session, user, galaxy_session): cls = model.UserAction assert cls.__tablename__ == 'user_action' From d513e35e91411415e981e4d8c5df2dec34f45e4a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 12:00:17 -0400 Subject: [PATCH 053/225] Map HistoryTagAssoc declaratively (see note) Also, rename relationship: 'histories' > 'history' (there's only one) --- lib/galaxy/model/__init__.py | 14 ++++++++++++-- lib/galaxy/model/mapping.py | 22 +++++++--------------- lib/galaxy/model/tags.py | 2 +- test/unit/model/test_model.py | 9 +++++++-- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 2d1db2ab051..d7a275d4c55 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6791,8 +6791,18 @@ class ItemTagAssociation(Dictifiable): return new_ta -class HistoryTagAssociation(ItemTagAssociation, RepresentById): - pass +class HistoryTagAssociation(Base, ItemTagAssociation, RepresentById): + __tablename__ = 'history_tag_association' + id = Column('id', Integer, primary_key=True) + history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) + tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column('user_tname', TrimmedString(255), index=True) + value = Column('value', TrimmedString(255), index=True) + user_value = Column('user_value', TrimmedString(255), index=True) + history = relationship('History', back_populates='tags') + tag = relationship('Tag', back_populates='tagged_histories') + user = relationship('User') class HistoryDatasetAssociationTagAssociation(ItemTagAssociation, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index a7331d7c90b..4f161f21c5b 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1115,16 +1115,6 @@ model.Tag.table = Table( Column("name", TrimmedString(255)), UniqueConstraint("name")) -model.HistoryTagAssociation.table = Table( - "history_tag_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("tag_id", Integer, ForeignKey("tag.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("user_tname", TrimmedString(255), index=True), - Column("value", TrimmedString(255), index=True), - Column("user_value", TrimmedString(255), index=True)) - model.HistoryDatasetAssociationTagAssociation.table = Table( "history_dataset_association_tag_association", metadata, Column("id", Integer, primary_key=True), @@ -1448,8 +1438,8 @@ mapper_registry.map_imperatively(model.History, model.History.table, properties= order_by=asc(model.HistoryDatasetCollectionAssociation.table.c.hid), viewonly=True), tags=relation(model.HistoryTagAssociation, - order_by=model.HistoryTagAssociation.table.c.id, - backref="histories"), + order_by=model.HistoryTagAssociation.id, + back_populates="history"), annotations=relation(model.HistoryAnnotationAssociation, order_by=model.HistoryAnnotationAssociation.id, back_populates="history"), @@ -2224,15 +2214,17 @@ mapper_registry.map_imperatively(model.VisualizationUserShareAssociation, model. )) # Tag tables. -simple_mapping(model.Tag, - children=relation(model.Tag, backref=backref('parent', remote_side=[model.Tag.table.c.id]))) + +mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( + children=relation(model.Tag, backref=backref('parent', remote_side=[model.Tag.table.c.id])), + tagged_histories=relation(model.HistoryTagAssociation, back_populates='tag') +)) def tag_mapping(tag_association_class, backref_name): simple_mapping(tag_association_class, tag=relation(model.Tag, backref=backref_name), user=relation(model.User)) -tag_mapping(model.HistoryTagAssociation, "tagged_histories") tag_mapping(model.HistoryDatasetAssociationTagAssociation, "tagged_history_dataset_associations") tag_mapping(model.LibraryDatasetDatasetAssociationTagAssociation, "tagged_library_dataset_dataset_associations") tag_mapping(model.PageTagAssociation, "tagged_pages") diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 14cf79c4d1c..7fe98887926 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -358,7 +358,7 @@ class GalaxyTagHandler(TagHandler): TagHandler.__init__(self, sa_session) self.item_tag_assoc_info["History"] = ItemTagAssocInfo(model.History, model.HistoryTagAssociation, - model.HistoryTagAssociation.table.c.history_id) + model.HistoryTagAssociation.history_id) self.item_tag_assoc_info["HistoryDatasetAssociation"] = \ ItemTagAssocInfo(model.HistoryDatasetAssociation, model.HistoryDatasetAssociationTagAssociation, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 11952c9001f..a456ab78926 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -328,7 +328,7 @@ def test_HistoryRatingAssociation(model, session, history, user): def test_HistoryTagAssociation(model, session, history, tag, user): cls = model.HistoryTagAssociation - # TODO assert cls.__tablename__ == '' + assert cls.__tablename__ == 'history_tag_association' with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -337,13 +337,18 @@ def test_HistoryTagAssociation(model, session, history, tag, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.history_id == history.id assert stored_obj.tag_id == tag.id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value + # test mapped relationships + assert stored_obj.history.id == history.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id def test_LibraryDatasetCollectionAnnotationAssociation( From 644f4b68ee6d00ea10b455b98ddb974f37e226c0 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 12:33:41 -0400 Subject: [PATCH 054/225] Map HDATagAssoc declaratively (see note) Also, rename incorrectly named relationship: HDA.tags: the correct reverse is `history_dataset_association`, NOT `history_tag_associations`. --- lib/galaxy/model/__init__.py | 14 ++++++++++++-- lib/galaxy/model/mapping.py | 18 ++++-------------- lib/galaxy/model/tags.py | 2 +- test/unit/model/test_model.py | 9 +++++++-- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index d7a275d4c55..4b8113aeb18 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6805,8 +6805,18 @@ class HistoryTagAssociation(Base, ItemTagAssociation, RepresentById): user = relationship('User') -class HistoryDatasetAssociationTagAssociation(ItemTagAssociation, RepresentById): - pass +class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, RepresentById): + __tablename__ = 'history_dataset_association_tag_association' + id = Column('id', Integer, primary_key=True) + history_dataset_association_id = Column("history_dataset_association_id", Integer, ForeignKey("history_dataset_association.id"), index=True) + tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column('user_tname', TrimmedString(255), index=True) + value = Column('value', TrimmedString(255), index=True) + user_value = Column('user_value', TrimmedString(255), index=True) + history_dataset_association = relationship('HistoryDatasetAssociation', back_populates='tags') + tag = relationship('Tag', back_populates='tagged_history_dataset_associations') + user = relationship('User') class LibraryDatasetDatasetAssociationTagAssociation(ItemTagAssociation, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 4f161f21c5b..9a7c029ee56 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1115,16 +1115,6 @@ model.Tag.table = Table( Column("name", TrimmedString(255)), UniqueConstraint("name")) -model.HistoryDatasetAssociationTagAssociation.table = Table( - "history_dataset_association_tag_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_dataset_association_id", Integer, ForeignKey("history_dataset_association.id"), index=True), - Column("tag_id", Integer, ForeignKey("tag.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("user_tname", TrimmedString(255), index=True), - Column("value", TrimmedString(255), index=True), - Column("user_value", TrimmedString(255), index=True)) - model.LibraryDatasetDatasetAssociationTagAssociation.table = Table( "library_dataset_dataset_association_tag_association", metadata, Column("id", Integer, primary_key=True), @@ -1316,8 +1306,8 @@ simple_mapping(model.HistoryDatasetAssociation, == model.LibraryDatasetDatasetAssociation.table.c.copied_from_history_dataset_association_id), backref='copied_from_history_dataset_association'), tags=relation(model.HistoryDatasetAssociationTagAssociation, - order_by=model.HistoryDatasetAssociationTagAssociation.table.c.id, - backref='history_tag_associations'), + order_by=model.HistoryDatasetAssociationTagAssociation.id, + back_populates='history_dataset_association'), annotations=relation(model.HistoryDatasetAssociationAnnotationAssociation, order_by=model.HistoryDatasetAssociationAnnotationAssociation.id, back_populates="hda"), @@ -2217,7 +2207,8 @@ mapper_registry.map_imperatively(model.VisualizationUserShareAssociation, model. mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( children=relation(model.Tag, backref=backref('parent', remote_side=[model.Tag.table.c.id])), - tagged_histories=relation(model.HistoryTagAssociation, back_populates='tag') + tagged_histories=relation(model.HistoryTagAssociation, back_populates='tag'), + tagged_history_dataset_associations=relation(model.HistoryDatasetAssociationTagAssociation, back_populates='tag'), )) @@ -2225,7 +2216,6 @@ def tag_mapping(tag_association_class, backref_name): simple_mapping(tag_association_class, tag=relation(model.Tag, backref=backref_name), user=relation(model.User)) -tag_mapping(model.HistoryDatasetAssociationTagAssociation, "tagged_history_dataset_associations") tag_mapping(model.LibraryDatasetDatasetAssociationTagAssociation, "tagged_library_dataset_dataset_associations") tag_mapping(model.PageTagAssociation, "tagged_pages") tag_mapping(model.StoredWorkflowTagAssociation, "tagged_workflows") diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 7fe98887926..077f5b1be22 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -362,7 +362,7 @@ class GalaxyTagHandler(TagHandler): self.item_tag_assoc_info["HistoryDatasetAssociation"] = \ ItemTagAssocInfo(model.HistoryDatasetAssociation, model.HistoryDatasetAssociationTagAssociation, - model.HistoryDatasetAssociationTagAssociation.table.c.history_dataset_association_id) + model.HistoryDatasetAssociationTagAssociation.history_dataset_association_id) self.item_tag_assoc_info["HistoryDatasetCollectionAssociation"] = \ ItemTagAssocInfo(model.HistoryDatasetCollectionAssociation, model.HistoryDatasetCollectionTagAssociation, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index a456ab78926..13cbce47da3 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -238,7 +238,7 @@ def test_HistoryDatasetAssociationRatingAssociation(model, session, history_data def test_HistoryDatasetAssociationTagAssociation( model, session, history_dataset_association, tag, user): cls = model.HistoryDatasetAssociationTagAssociation - # TODO assert cls.__tablename__ == '' + assert cls.__tablename__ == 'history_dataset_association_tag_association' with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -247,13 +247,18 @@ def test_HistoryDatasetAssociationTagAssociation( obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.history_dataset_association_id == history_dataset_association.id assert stored_obj.tag_id == tag.id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value + # test mapped relationships + assert stored_obj.history_dataset_association.id == history_dataset_association.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id def test_HistoryDatasetCollectionAssociationAnnotationAssociation( From 1ad509a3fa3cf5b0ec5e293b77986ced6f6c92b4 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 14:15:48 -0400 Subject: [PATCH 055/225] Map LDDATagAssoc declaratively (see note) Rename incorrectly named relationship: LDDA.tags: reverse relationship is `library_dataset_dataset_association`, NOT `history_tag_associations`. --- lib/galaxy/model/__init__.py | 20 +++++++++++++++++--- lib/galaxy/model/mapping.py | 16 +++------------- lib/galaxy/model/tags.py | 2 +- test/unit/model/test_model.py | 9 +++++++-- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4b8113aeb18..89c6fe1ddf5 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6808,7 +6808,8 @@ class HistoryTagAssociation(Base, ItemTagAssociation, RepresentById): class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'history_dataset_association_tag_association' id = Column('id', Integer, primary_key=True) - history_dataset_association_id = Column("history_dataset_association_id", Integer, ForeignKey("history_dataset_association.id"), index=True) + history_dataset_association_id = Column('history_dataset_association_id', Integer, + ForeignKey('history_dataset_association.id'), index=True) tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) user_tname = Column('user_tname', TrimmedString(255), index=True) @@ -6819,8 +6820,21 @@ class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, Represen user = relationship('User') -class LibraryDatasetDatasetAssociationTagAssociation(ItemTagAssociation, RepresentById): - pass +class LibraryDatasetDatasetAssociationTagAssociation(Base, ItemTagAssociation, RepresentById): + __tablename__ = 'library_dataset_dataset_association_tag_association' + id = Column('id', Integer, primary_key=True) + library_dataset_dataset_association_id = Column('library_dataset_dataset_association_id', + Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column('user_tname', TrimmedString(255), index=True) + value = Column('value', TrimmedString(255), index=True) + user_value = Column('user_value', TrimmedString(255), index=True) + + library_dataset_dataset_association = relationship( + 'LibraryDatasetDatasetAssociation', back_populates='tags') + tag = relationship('Tag', back_populates='tagged_library_dataset_dataset_associations') + user = relationship('User') class PageTagAssociation(ItemTagAssociation, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 9a7c029ee56..af2772fb833 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1115,16 +1115,6 @@ model.Tag.table = Table( Column("name", TrimmedString(255)), UniqueConstraint("name")) -model.LibraryDatasetDatasetAssociationTagAssociation.table = Table( - "library_dataset_dataset_association_tag_association", metadata, - Column("id", Integer, primary_key=True), - Column("library_dataset_dataset_association_id", Integer, ForeignKey("library_dataset_dataset_association.id"), index=True), - Column("tag_id", Integer, ForeignKey("tag.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("user_tname", TrimmedString(255), index=True), - Column("value", TrimmedString(255), index=True), - Column("user_value", TrimmedString(255), index=True)) - model.StoredWorkflowTagAssociation.table = Table( "stored_workflow_tag_association", metadata, Column("id", Integer, primary_key=True), @@ -1657,8 +1647,8 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.L == model.LibraryDatasetDatasetAssociation.table.c.id), backref='parent_ldda'), tags=relation(model.LibraryDatasetDatasetAssociationTagAssociation, - order_by=model.LibraryDatasetDatasetAssociationTagAssociation.table.c.id, - backref='history_tag_associations'), + order_by=model.LibraryDatasetDatasetAssociationTagAssociation.id, + back_populates='library_dataset_dataset_association'), extended_metadata=relation(model.ExtendedMetadata, primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.extended_metadata_id == model.ExtendedMetadata.table.c.id) ), @@ -2209,6 +2199,7 @@ mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( children=relation(model.Tag, backref=backref('parent', remote_side=[model.Tag.table.c.id])), tagged_histories=relation(model.HistoryTagAssociation, back_populates='tag'), tagged_history_dataset_associations=relation(model.HistoryDatasetAssociationTagAssociation, back_populates='tag'), + tagged_library_dataset_dataset_associations=relation(model.LibraryDatasetDatasetAssociationTagAssociation, back_populates='tag'), )) @@ -2216,7 +2207,6 @@ def tag_mapping(tag_association_class, backref_name): simple_mapping(tag_association_class, tag=relation(model.Tag, backref=backref_name), user=relation(model.User)) -tag_mapping(model.LibraryDatasetDatasetAssociationTagAssociation, "tagged_library_dataset_dataset_associations") tag_mapping(model.PageTagAssociation, "tagged_pages") tag_mapping(model.StoredWorkflowTagAssociation, "tagged_workflows") tag_mapping(model.WorkflowStepTagAssociation, "tagged_workflow_steps") diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 077f5b1be22..5b0c8cc6abc 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -370,7 +370,7 @@ class GalaxyTagHandler(TagHandler): self.item_tag_assoc_info["LibraryDatasetDatasetAssociation"] = \ ItemTagAssocInfo(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssociationTagAssociation, - model.LibraryDatasetDatasetAssociationTagAssociation.table.c.library_dataset_dataset_association_id) + model.LibraryDatasetDatasetAssociationTagAssociation.library_dataset_dataset_association_id) self.item_tag_assoc_info["Page"] = ItemTagAssocInfo(model.Page, model.PageTagAssociation, model.PageTagAssociation.table.c.page_id) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 13cbce47da3..63aad641652 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -414,7 +414,7 @@ def test_LibraryDatasetCollectionRatingAssociation( def test_LibraryDatasetDatasetAssociationTagAssociation( model, session, library_dataset_dataset_association, tag, user): cls = model.LibraryDatasetDatasetAssociationTagAssociation - # TODO assert cls.__tablename__ == '' + assert cls.__tablename__ == 'library_dataset_dataset_association_tag_association' with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -423,13 +423,18 @@ def test_LibraryDatasetDatasetAssociationTagAssociation( obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id assert stored_obj.tag_id == tag.id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value + # test mapped relationships + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id def test_PageAnnotationAssociation(model, session, page, user): From b082f493c23d4ffe3952b74e12007ecb30558e80 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 14:23:31 -0400 Subject: [PATCH 056/225] Map PageTagAssociation declaratively (see note) Rename relationship: pages >> page (it's one page) --- lib/galaxy/model/__init__.py | 15 +++++++++++++-- lib/galaxy/model/mapping.py | 16 +++------------- lib/galaxy/model/tags.py | 2 +- test/unit/model/test_model.py | 9 +++++++-- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 89c6fe1ddf5..c90596d306e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6837,8 +6837,19 @@ class LibraryDatasetDatasetAssociationTagAssociation(Base, ItemTagAssociation, R user = relationship('User') -class PageTagAssociation(ItemTagAssociation, RepresentById): - pass +class PageTagAssociation(Base, ItemTagAssociation, RepresentById): + __tablename__ = 'page_tag_association' + id = Column('id', Integer, primary_key=True) + page_id = Column('page_id', Integer, ForeignKey('page.id'), index=True) + tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column('user_tname', TrimmedString(255), index=True) + value = Column('value', TrimmedString(255), index=True) + user_value = Column('user_value', TrimmedString(255), index=True) + page = relationship('Page', back_populates='tags') + tag = relationship('Tag', back_populates='tagged_pages') + user = relationship('User') + class WorkflowStepTagAssociation(ItemTagAssociation, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index af2772fb833..b9a02d150e7 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1125,16 +1125,6 @@ model.StoredWorkflowTagAssociation.table = Table( Column("value", Unicode(255), index=True), Column("user_value", Unicode(255), index=True)) -model.PageTagAssociation.table = Table( - "page_tag_association", metadata, - Column("id", Integer, primary_key=True), - Column("page_id", Integer, ForeignKey("page.id"), index=True), - Column("tag_id", Integer, ForeignKey("tag.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("user_tname", TrimmedString(255), index=True), - Column("value", TrimmedString(255), index=True), - Column("user_value", TrimmedString(255), index=True)) - model.WorkflowStepTagAssociation.table = Table( "workflow_step_tag_association", metadata, Column("id", Integer, primary_key=True), @@ -2136,8 +2126,8 @@ mapper_registry.map_imperatively(model.Page, model.Page.table, properties=dict( primaryjoin=(model.Page.table.c.latest_revision_id == model.PageRevision.id), lazy=False), tags=relation(model.PageTagAssociation, - order_by=model.PageTagAssociation.table.c.id, - backref="pages"), + order_by=model.PageTagAssociation.id, + back_populates="page"), annotations=relation(model.PageAnnotationAssociation, order_by=model.PageAnnotationAssociation.id, back_populates="page"), @@ -2200,6 +2190,7 @@ mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( tagged_histories=relation(model.HistoryTagAssociation, back_populates='tag'), tagged_history_dataset_associations=relation(model.HistoryDatasetAssociationTagAssociation, back_populates='tag'), tagged_library_dataset_dataset_associations=relation(model.LibraryDatasetDatasetAssociationTagAssociation, back_populates='tag'), + tagged_pages=relation(model.PageTagAssociation, back_populates='tag'), )) @@ -2207,7 +2198,6 @@ def tag_mapping(tag_association_class, backref_name): simple_mapping(tag_association_class, tag=relation(model.Tag, backref=backref_name), user=relation(model.User)) -tag_mapping(model.PageTagAssociation, "tagged_pages") tag_mapping(model.StoredWorkflowTagAssociation, "tagged_workflows") tag_mapping(model.WorkflowStepTagAssociation, "tagged_workflow_steps") tag_mapping(model.VisualizationTagAssociation, "tagged_visualizations") diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 5b0c8cc6abc..b426fcb8cf9 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -373,7 +373,7 @@ class GalaxyTagHandler(TagHandler): model.LibraryDatasetDatasetAssociationTagAssociation.library_dataset_dataset_association_id) self.item_tag_assoc_info["Page"] = ItemTagAssocInfo(model.Page, model.PageTagAssociation, - model.PageTagAssociation.table.c.page_id) + model.PageTagAssociation.page_id) self.item_tag_assoc_info["StoredWorkflow"] = ItemTagAssocInfo(model.StoredWorkflow, model.StoredWorkflowTagAssociation, model.StoredWorkflowTagAssociation.table.c.stored_workflow_id) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 63aad641652..84128b41dec 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -492,7 +492,7 @@ def test_PageRevision(model, session, page): def test_PageTagAssociation(model, session, page, tag, user): cls = model.PageTagAssociation - # TODO assert cls.__tablename__ == '' + assert cls.__tablename__ == 'page_tag_association' with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -501,13 +501,18 @@ def test_PageTagAssociation(model, session, page, tag, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.page_id == page.id assert stored_obj.tag_id == tag.id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value + # test mapped relationships + assert stored_obj.page.id == page.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id def test_PageUserShareAssociation(model, session, page, user): From e9620842c5f1f2f7f2ec52c118a76eb917d80728 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 14:37:57 -0400 Subject: [PATCH 057/225] Map WFSTagAssociation declaratively (see note) Also rename relationship: pages > page (only one) --- lib/galaxy/model/__init__.py | 15 +++++++++++++-- lib/galaxy/model/mapping.py | 16 +++------------- test/unit/model/test_model.py | 9 +++++++-- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index c90596d306e..8191e2aa907 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6851,9 +6851,20 @@ class PageTagAssociation(Base, ItemTagAssociation, RepresentById): user = relationship('User') +class WorkflowStepTagAssociation(Base, ItemTagAssociation, RepresentById): + __tablename__ = 'workflow_step_tag_association' + id = Column('id', Integer, primary_key=True) -class WorkflowStepTagAssociation(ItemTagAssociation, RepresentById): - pass + workflow_step_id = Column('workflow_step_id', Integer, ForeignKey('workflow_step.id'), index=True) + tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column('user_tname', TrimmedString(255), index=True) + value = Column('value', TrimmedString(255), index=True) + user_value = Column('user_value', TrimmedString(255), index=True) + + workflow_step = relationship('WorkflowStep', back_populates='tags') + tag = relationship('Tag', back_populates='tagged_workflow_steps') + user = relationship('User') class StoredWorkflowTagAssociation(ItemTagAssociation, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index b9a02d150e7..adc9715068c 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1125,16 +1125,6 @@ model.StoredWorkflowTagAssociation.table = Table( Column("value", Unicode(255), index=True), Column("user_value", Unicode(255), index=True)) -model.WorkflowStepTagAssociation.table = Table( - "workflow_step_tag_association", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True), - Column("tag_id", Integer, ForeignKey("tag.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("user_tname", Unicode(255), index=True), - Column("value", Unicode(255), index=True), - Column("user_value", Unicode(255), index=True)) - model.VisualizationTagAssociation.table = Table( "visualization_tag_association", metadata, Column("id", Integer, primary_key=True), @@ -1913,8 +1903,8 @@ mapper_registry.map_imperatively(model.WorkflowStep, model.WorkflowStep.table, p primaryjoin=(model.DynamicTool.id == model.WorkflowStep.table.c.dynamic_tool_id), back_populates="workflow_steps"), tags=relation(model.WorkflowStepTagAssociation, - order_by=model.WorkflowStepTagAssociation.table.c.id, - backref="workflow_steps"), + order_by=model.WorkflowStepTagAssociation.id, + back_populates="workflow_step"), annotations=relation(model.WorkflowStepAnnotationAssociation, order_by=model.WorkflowStepAnnotationAssociation.id, back_populates="workflow_step") @@ -2191,6 +2181,7 @@ mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( tagged_history_dataset_associations=relation(model.HistoryDatasetAssociationTagAssociation, back_populates='tag'), tagged_library_dataset_dataset_associations=relation(model.LibraryDatasetDatasetAssociationTagAssociation, back_populates='tag'), tagged_pages=relation(model.PageTagAssociation, back_populates='tag'), + tagged_workflow_steps=relation(model.WorkflowStepTagAssociation, back_populates='tag'), )) @@ -2199,7 +2190,6 @@ def tag_mapping(tag_association_class, backref_name): tag_mapping(model.StoredWorkflowTagAssociation, "tagged_workflows") -tag_mapping(model.WorkflowStepTagAssociation, "tagged_workflow_steps") tag_mapping(model.VisualizationTagAssociation, "tagged_visualizations") tag_mapping(model.HistoryDatasetCollectionTagAssociation, "tagged_history_dataset_collections") tag_mapping(model.LibraryDatasetCollectionTagAssociation, "tagged_library_dataset_collections") diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 84128b41dec..13853ce2f77 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -892,7 +892,7 @@ def test_WorkflowStepAnnotationAssociation(model, session, workflow_step, user): def test_WorkflowStepTagAssociation(model, session, workflow_step, tag, user): cls = model.WorkflowStepTagAssociation - # TODO assert cls.__tablename__ == '' + assert cls.__tablename__ == 'workflow_step_tag_association' with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -901,13 +901,18 @@ def test_WorkflowStepTagAssociation(model, session, workflow_step, tag, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.workflow_step_id == workflow_step.id assert stored_obj.tag_id == tag.id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value + # test mapped relationships + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id def test_VisualizationTagAssociation(model, session, visualization, tag, user): From 94e0bb47ff6b4d160d51601db96fe54676f7c497 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 14:53:39 -0400 Subject: [PATCH 058/225] Map StoredWorkflowTagAssoc (see note) - Rename relationship: stored_workflows > stored_workflow - Rename reverse relationship: tagged_workflows > tagged_stored_workflows: - for consistency + if we decide to tag workflows, this would get in the way. --- lib/galaxy/model/__init__.py | 23 ++++++++++++++++++----- lib/galaxy/model/mapping.py | 22 ++++++---------------- lib/galaxy/model/tags.py | 2 +- test/unit/model/test_model.py | 9 +++++++-- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 8191e2aa907..4e88dc8b354 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6793,6 +6793,7 @@ class ItemTagAssociation(Dictifiable): class HistoryTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'history_tag_association' + id = Column('id', Integer, primary_key=True) history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) @@ -6807,6 +6808,7 @@ class HistoryTagAssociation(Base, ItemTagAssociation, RepresentById): class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'history_dataset_association_tag_association' + id = Column('id', Integer, primary_key=True) history_dataset_association_id = Column('history_dataset_association_id', Integer, ForeignKey('history_dataset_association.id'), index=True) @@ -6822,6 +6824,7 @@ class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, Represen class LibraryDatasetDatasetAssociationTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'library_dataset_dataset_association_tag_association' + id = Column('id', Integer, primary_key=True) library_dataset_dataset_association_id = Column('library_dataset_dataset_association_id', Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) @@ -6830,7 +6833,6 @@ class LibraryDatasetDatasetAssociationTagAssociation(Base, ItemTagAssociation, R user_tname = Column('user_tname', TrimmedString(255), index=True) value = Column('value', TrimmedString(255), index=True) user_value = Column('user_value', TrimmedString(255), index=True) - library_dataset_dataset_association = relationship( 'LibraryDatasetDatasetAssociation', back_populates='tags') tag = relationship('Tag', back_populates='tagged_library_dataset_dataset_associations') @@ -6839,6 +6841,7 @@ class LibraryDatasetDatasetAssociationTagAssociation(Base, ItemTagAssociation, R class PageTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'page_tag_association' + id = Column('id', Integer, primary_key=True) page_id = Column('page_id', Integer, ForeignKey('page.id'), index=True) tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) @@ -6853,22 +6856,32 @@ class PageTagAssociation(Base, ItemTagAssociation, RepresentById): class WorkflowStepTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'workflow_step_tag_association' + id = Column('id', Integer, primary_key=True) - workflow_step_id = Column('workflow_step_id', Integer, ForeignKey('workflow_step.id'), index=True) tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) user_tname = Column('user_tname', TrimmedString(255), index=True) value = Column('value', TrimmedString(255), index=True) user_value = Column('user_value', TrimmedString(255), index=True) - workflow_step = relationship('WorkflowStep', back_populates='tags') tag = relationship('Tag', back_populates='tagged_workflow_steps') user = relationship('User') -class StoredWorkflowTagAssociation(ItemTagAssociation, RepresentById): - pass +class StoredWorkflowTagAssociation(Base, ItemTagAssociation, RepresentById): + __tablename__ = 'stored_workflow_tag_association' + + id = Column('id', Integer, primary_key=True) + stored_workflow_id = Column('stored_workflow_id', Integer, ForeignKey('stored_workflow.id'), index=True) + tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column('user_tname', TrimmedString(255), index=True) + value = Column('value', TrimmedString(255), index=True) + user_value = Column('user_value', TrimmedString(255), index=True) + stored_workflow = relationship('StoredWorkflow', back_populates='tags') + tag = relationship('Tag', back_populates='tagged_stored_workflows') + user = relationship('User') class VisualizationTagAssociation(ItemTagAssociation, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index adc9715068c..cc1940f6007 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1115,16 +1115,6 @@ model.Tag.table = Table( Column("name", TrimmedString(255)), UniqueConstraint("name")) -model.StoredWorkflowTagAssociation.table = Table( - "stored_workflow_tag_association", metadata, - Column("id", Integer, primary_key=True), - Column("stored_workflow_id", Integer, ForeignKey("stored_workflow.id"), index=True), - Column("tag_id", Integer, ForeignKey("tag.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("user_tname", Unicode(255), index=True), - Column("value", Unicode(255), index=True), - Column("user_value", Unicode(255), index=True)) - model.VisualizationTagAssociation.table = Table( "visualization_tag_association", metadata, Column("id", Integer, primary_key=True), @@ -1953,15 +1943,15 @@ mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.tabl primaryjoin=(model.StoredWorkflow.table.c.latest_workflow_id == model.Workflow.table.c.id), lazy=False), tags=relation(model.StoredWorkflowTagAssociation, - order_by=model.StoredWorkflowTagAssociation.table.c.id, - backref="stored_workflows"), + order_by=model.StoredWorkflowTagAssociation.id, + back_populates="stored_workflow"), owner_tags=relation(model.StoredWorkflowTagAssociation, primaryjoin=( - and_(model.StoredWorkflow.table.c.id == model.StoredWorkflowTagAssociation.table.c.stored_workflow_id, - model.StoredWorkflow.table.c.user_id == model.StoredWorkflowTagAssociation.table.c.user_id) + and_(model.StoredWorkflow.table.c.id == model.StoredWorkflowTagAssociation.stored_workflow_id, + model.StoredWorkflow.table.c.user_id == model.StoredWorkflowTagAssociation.user_id) ), viewonly=True, - order_by=model.StoredWorkflowTagAssociation.table.c.id), + order_by=model.StoredWorkflowTagAssociation.id), annotations=relation(model.StoredWorkflowAnnotationAssociation, order_by=model.StoredWorkflowAnnotationAssociation.id, back_populates="stored_workflow"), @@ -2182,6 +2172,7 @@ mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( tagged_library_dataset_dataset_associations=relation(model.LibraryDatasetDatasetAssociationTagAssociation, back_populates='tag'), tagged_pages=relation(model.PageTagAssociation, back_populates='tag'), tagged_workflow_steps=relation(model.WorkflowStepTagAssociation, back_populates='tag'), + tagged_stored_workflows=relation(model.StoredWorkflowTagAssociation, back_populates='tag'), )) @@ -2189,7 +2180,6 @@ def tag_mapping(tag_association_class, backref_name): simple_mapping(tag_association_class, tag=relation(model.Tag, backref=backref_name), user=relation(model.User)) -tag_mapping(model.StoredWorkflowTagAssociation, "tagged_workflows") tag_mapping(model.VisualizationTagAssociation, "tagged_visualizations") tag_mapping(model.HistoryDatasetCollectionTagAssociation, "tagged_history_dataset_collections") tag_mapping(model.LibraryDatasetCollectionTagAssociation, "tagged_library_dataset_collections") diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index b426fcb8cf9..9d38ac8fa75 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -376,7 +376,7 @@ class GalaxyTagHandler(TagHandler): model.PageTagAssociation.page_id) self.item_tag_assoc_info["StoredWorkflow"] = ItemTagAssocInfo(model.StoredWorkflow, model.StoredWorkflowTagAssociation, - model.StoredWorkflowTagAssociation.table.c.stored_workflow_id) + model.StoredWorkflowTagAssociation.stored_workflow_id) self.item_tag_assoc_info["Visualization"] = ItemTagAssocInfo(model.Visualization, model.VisualizationTagAssociation, model.VisualizationTagAssociation.table.c.visualization_id) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 13853ce2f77..b4cf573b868 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -667,7 +667,7 @@ def test_StoredWorkflowRatingAssociation(model, session, stored_workflow, user): def test_StoredWorkflowTagAssociation(model, session, stored_workflow, tag, user): cls = model.StoredWorkflowTagAssociation - # TODO assert cls.__tablename__ == '' + assert cls.__tablename__ == 'stored_workflow_tag_association' with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -676,13 +676,18 @@ def test_StoredWorkflowTagAssociation(model, session, stored_workflow, tag, user obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.stored_workflow_id == stored_workflow.id assert stored_obj.tag_id == tag.id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value + # test mapped relationships + assert stored_obj.stored_workflow.id == stored_workflow.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id def test_ToolTagAssociation(model, session, tag, user): From 7cd50fb71abeee9c4cdff2eef5a2bd8a8b4fc362 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 15:00:06 -0400 Subject: [PATCH 059/225] Map VisualizationTagAssoc declaratively (see note) Rename relationship: visualizations > visualization (there's only one) --- lib/galaxy/model/__init__.py | 15 +++++++++++++-- lib/galaxy/model/mapping.py | 16 +++------------- lib/galaxy/model/tags.py | 2 +- test/unit/model/test_model.py | 9 +++++++-- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4e88dc8b354..cb0b56b44bd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6884,8 +6884,19 @@ class StoredWorkflowTagAssociation(Base, ItemTagAssociation, RepresentById): user = relationship('User') -class VisualizationTagAssociation(ItemTagAssociation, RepresentById): - pass +class VisualizationTagAssociation(Base, ItemTagAssociation, RepresentById): + __tablename__ = 'visualization_tag_association' + + id = Column('id', Integer, primary_key=True) + visualization_id = Column('visualization_id', Integer, ForeignKey('visualization.id'), index=True) + tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column('user_tname', TrimmedString(255), index=True) + value = Column('value', TrimmedString(255), index=True) + user_value = Column('user_value', TrimmedString(255), index=True) + visualization = relationship('Visualization', back_populates='tags') + tag = relationship('Tag', back_populates='tagged_visualizations') + user = relationship('User') class HistoryDatasetCollectionTagAssociation(ItemTagAssociation, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index cc1940f6007..0c24c1462f5 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1115,16 +1115,6 @@ model.Tag.table = Table( Column("name", TrimmedString(255)), UniqueConstraint("name")) -model.VisualizationTagAssociation.table = Table( - "visualization_tag_association", metadata, - Column("id", Integer, primary_key=True), - Column("visualization_id", Integer, ForeignKey("visualization.id"), index=True), - Column("tag_id", Integer, ForeignKey("tag.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("user_tname", TrimmedString(255), index=True), - Column("value", TrimmedString(255), index=True), - Column("user_value", TrimmedString(255), index=True)) - model.HistoryDatasetCollectionTagAssociation.table = Table( "history_dataset_collection_tag_association", metadata, Column("id", Integer, primary_key=True), @@ -2137,8 +2127,8 @@ mapper_registry.map_imperatively(model.Visualization, model.Visualization.table, primaryjoin=(model.Visualization.table.c.latest_revision_id == model.VisualizationRevision.id), lazy=False), tags=relation(model.VisualizationTagAssociation, - order_by=model.VisualizationTagAssociation.table.c.id, - backref="visualizations"), + order_by=model.VisualizationTagAssociation.id, + back_populates="visualization"), annotations=relation(model.VisualizationAnnotationAssociation, order_by=model.VisualizationAnnotationAssociation.id, back_populates="visualization"), @@ -2173,6 +2163,7 @@ mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( tagged_pages=relation(model.PageTagAssociation, back_populates='tag'), tagged_workflow_steps=relation(model.WorkflowStepTagAssociation, back_populates='tag'), tagged_stored_workflows=relation(model.StoredWorkflowTagAssociation, back_populates='tag'), + tagged_visualizations=relation(model.VisualizationTagAssociation, back_populates='tag'), )) @@ -2180,7 +2171,6 @@ def tag_mapping(tag_association_class, backref_name): simple_mapping(tag_association_class, tag=relation(model.Tag, backref=backref_name), user=relation(model.User)) -tag_mapping(model.VisualizationTagAssociation, "tagged_visualizations") tag_mapping(model.HistoryDatasetCollectionTagAssociation, "tagged_history_dataset_collections") tag_mapping(model.LibraryDatasetCollectionTagAssociation, "tagged_library_dataset_collections") tag_mapping(model.ToolTagAssociation, "tagged_tools") diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index 9d38ac8fa75..f77a20dd250 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -379,7 +379,7 @@ class GalaxyTagHandler(TagHandler): model.StoredWorkflowTagAssociation.stored_workflow_id) self.item_tag_assoc_info["Visualization"] = ItemTagAssocInfo(model.Visualization, model.VisualizationTagAssociation, - model.VisualizationTagAssociation.table.c.visualization_id) + model.VisualizationTagAssociation.visualization_id) class GalaxyTagHandlerSession(GalaxyTagHandler): diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index b4cf573b868..90d58b883b5 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -922,7 +922,7 @@ def test_WorkflowStepTagAssociation(model, session, workflow_step, tag, user): def test_VisualizationTagAssociation(model, session, visualization, tag, user): cls = model.VisualizationTagAssociation - # TODO assert cls.__tablename__ == '' + assert cls.__tablename__ == 'visualization_tag_association' with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -931,13 +931,18 @@ def test_VisualizationTagAssociation(model, session, visualization, tag, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.visualization_id == visualization.id assert stored_obj.tag_id == tag.id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value + # test mapped relationships + assert stored_obj.visualization.id == visualization.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id @pytest.fixture(scope='module') From 00553014557f43dd3ab5cef26d67969492ba4d20 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 15:10:23 -0400 Subject: [PATCH 060/225] Map HDCTagAssoc declaratively (see note) Rename relationship: dataset_collections > dataset_collection (there's only one) --- lib/galaxy/model/__init__.py | 17 +++++++++++++++-- lib/galaxy/model/mapping.py | 17 +++-------------- lib/galaxy/model/tags.py | 2 +- test/unit/model/test_model.py | 9 +++++++-- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index cb0b56b44bd..c075a2f6f2a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6899,8 +6899,21 @@ class VisualizationTagAssociation(Base, ItemTagAssociation, RepresentById): user = relationship('User') -class HistoryDatasetCollectionTagAssociation(ItemTagAssociation, RepresentById): - pass +class HistoryDatasetCollectionTagAssociation(Base, ItemTagAssociation, RepresentById): + __tablename__ = 'history_dataset_collection_tag_association' + + id = Column('id', Integer, primary_key=True) + history_dataset_collection_id = Column('history_dataset_collection_id', Integer, + ForeignKey('history_dataset_collection_association.id'), index=True) + tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column('user_tname', TrimmedString(255), index=True) + value = Column('value', TrimmedString(255), index=True) + user_value = Column('user_value', TrimmedString(255), index=True) + dataset_collection = relationship('HistoryDatasetCollectionAssociation', back_populates='tags') + tag = relationship('Tag', back_populates='tagged_history_dataset_collections') + user = relationship('User') + class LibraryDatasetCollectionTagAssociation(ItemTagAssociation, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 0c24c1462f5..6899c15946d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1115,17 +1115,6 @@ model.Tag.table = Table( Column("name", TrimmedString(255)), UniqueConstraint("name")) -model.HistoryDatasetCollectionTagAssociation.table = Table( - "history_dataset_collection_tag_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_dataset_collection_id", Integer, - ForeignKey("history_dataset_collection_association.id"), index=True), - Column("tag_id", Integer, ForeignKey("tag.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("user_tname", TrimmedString(255), index=True), - Column("value", TrimmedString(255), index=True), - Column("user_value", TrimmedString(255), index=True)) - model.LibraryDatasetCollectionTagAssociation.table = Table( "library_dataset_collection_tag_association", metadata, Column("id", Integer, primary_key=True), @@ -1813,8 +1802,8 @@ simple_mapping(model.HistoryDatasetCollectionAssociation, uselist=False ), tags=relation(model.HistoryDatasetCollectionTagAssociation, - order_by=model.HistoryDatasetCollectionTagAssociation.table.c.id, - backref='dataset_collections'), + order_by=model.HistoryDatasetCollectionTagAssociation.id, + back_populates='dataset_collection'), annotations=relation(model.HistoryDatasetCollectionAssociationAnnotationAssociation, order_by=model.HistoryDatasetCollectionAssociationAnnotationAssociation.id, back_populates="history_dataset_collection"), @@ -2164,6 +2153,7 @@ mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( tagged_workflow_steps=relation(model.WorkflowStepTagAssociation, back_populates='tag'), tagged_stored_workflows=relation(model.StoredWorkflowTagAssociation, back_populates='tag'), tagged_visualizations=relation(model.VisualizationTagAssociation, back_populates='tag'), + tagged_history_dataset_collections=relation(model.HistoryDatasetCollectionTagAssociation, back_populates='tag'), )) @@ -2171,7 +2161,6 @@ def tag_mapping(tag_association_class, backref_name): simple_mapping(tag_association_class, tag=relation(model.Tag, backref=backref_name), user=relation(model.User)) -tag_mapping(model.HistoryDatasetCollectionTagAssociation, "tagged_history_dataset_collections") tag_mapping(model.LibraryDatasetCollectionTagAssociation, "tagged_library_dataset_collections") tag_mapping(model.ToolTagAssociation, "tagged_tools") diff --git a/lib/galaxy/model/tags.py b/lib/galaxy/model/tags.py index f77a20dd250..45c742c13a5 100644 --- a/lib/galaxy/model/tags.py +++ b/lib/galaxy/model/tags.py @@ -366,7 +366,7 @@ class GalaxyTagHandler(TagHandler): self.item_tag_assoc_info["HistoryDatasetCollectionAssociation"] = \ ItemTagAssocInfo(model.HistoryDatasetCollectionAssociation, model.HistoryDatasetCollectionTagAssociation, - model.HistoryDatasetCollectionTagAssociation.table.c.history_dataset_collection_id) + model.HistoryDatasetCollectionTagAssociation.history_dataset_collection_id) self.item_tag_assoc_info["LibraryDatasetDatasetAssociation"] = \ ItemTagAssocInfo(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssociationTagAssociation, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 90d58b883b5..7d3f432a49d 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -298,7 +298,7 @@ def test_HistoryDatasetCollectionRatingAssociation( def test_HistoryDatasetCollectionTagAssociation(model, session, history_dataset_collection_association, tag, user): cls = model.HistoryDatasetCollectionTagAssociation - # TODO assert cls.__tablename__ == '' + assert cls.__tablename__ == 'history_dataset_collection_tag_association' with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -307,13 +307,18 @@ def test_HistoryDatasetCollectionTagAssociation(model, session, history_dataset_ obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id assert stored_obj.tag_id == tag.id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value + # test mapped relationships + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id def test_HistoryRatingAssociation(model, session, history, user): From 97fcfe7a439787f2c209b41a402a1e05e1d69182 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 15:16:47 -0400 Subject: [PATCH 061/225] Map LDCTagAssoc declaratively (see note) Rename relationship: dataset_collections > dataset_collection (there's only one) --- lib/galaxy/model/__init__.py | 15 +++++++++++++-- lib/galaxy/model/mapping.py | 17 +++-------------- test/unit/model/test_model.py | 9 +++++++-- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index c075a2f6f2a..26765e004c2 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6915,9 +6915,20 @@ class HistoryDatasetCollectionTagAssociation(Base, ItemTagAssociation, Represent user = relationship('User') +class LibraryDatasetCollectionTagAssociation(Base, ItemTagAssociation, RepresentById): + __tablename__ = 'library_dataset_collection_tag_association' -class LibraryDatasetCollectionTagAssociation(ItemTagAssociation, RepresentById): - pass + id = Column('id', Integer, primary_key=True) + library_dataset_collection_id = Column('library_dataset_collection_id', Integer, + ForeignKey('library_dataset_collection_association.id'), index=True) + tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column('user_tname', TrimmedString(255), index=True) + value = Column('value', TrimmedString(255), index=True) + user_value = Column('user_value', TrimmedString(255), index=True) + dataset_collection = relationship('LibraryDatasetCollectionAssociation', back_populates='tags') + tag = relationship('Tag', back_populates='tagged_library_dataset_collections') + user = relationship('User') class ToolTagAssociation(ItemTagAssociation, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6899c15946d..1825b2613c8 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1115,17 +1115,6 @@ model.Tag.table = Table( Column("name", TrimmedString(255)), UniqueConstraint("name")) -model.LibraryDatasetCollectionTagAssociation.table = Table( - "library_dataset_collection_tag_association", metadata, - Column("id", Integer, primary_key=True), - Column("library_dataset_collection_id", Integer, - ForeignKey("library_dataset_collection_association.id"), index=True), - Column("tag_id", Integer, ForeignKey("tag.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("user_tname", TrimmedString(255), index=True), - Column("value", TrimmedString(255), index=True), - Column("user_value", TrimmedString(255), index=True)) - model.ToolTagAssociation.table = Table( "tool_tag_association", metadata, Column("id", Integer, primary_key=True), @@ -1817,8 +1806,8 @@ simple_mapping(model.LibraryDatasetCollectionAssociation, folder=relation(model.LibraryFolder, backref='dataset_collections'), tags=relation(model.LibraryDatasetCollectionTagAssociation, - order_by=model.LibraryDatasetCollectionTagAssociation.table.c.id, - backref='dataset_collections'), + order_by=model.LibraryDatasetCollectionTagAssociation.id, + back_populates='dataset_collection'), annotations=relation(model.LibraryDatasetCollectionAnnotationAssociation, order_by=model.LibraryDatasetCollectionAnnotationAssociation.id, back_populates="dataset_collection"), @@ -2154,6 +2143,7 @@ mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( tagged_stored_workflows=relation(model.StoredWorkflowTagAssociation, back_populates='tag'), tagged_visualizations=relation(model.VisualizationTagAssociation, back_populates='tag'), tagged_history_dataset_collections=relation(model.HistoryDatasetCollectionTagAssociation, back_populates='tag'), + tagged_library_dataset_collections=relation(model.LibraryDatasetCollectionTagAssociation, back_populates='tag'), )) @@ -2161,7 +2151,6 @@ def tag_mapping(tag_association_class, backref_name): simple_mapping(tag_association_class, tag=relation(model.Tag, backref=backref_name), user=relation(model.User)) -tag_mapping(model.LibraryDatasetCollectionTagAssociation, "tagged_library_dataset_collections") tag_mapping(model.ToolTagAssociation, "tagged_tools") mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 7d3f432a49d..ac196c0dbb6 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -382,7 +382,7 @@ def test_LibraryDatasetCollectionAnnotationAssociation( def test_LibraryDatasetCollectionTagAssociation(model, session, library_dataset_collection_association, tag, user): cls = model.LibraryDatasetCollectionTagAssociation - # TODO assert cls.__tablename__ == '' + assert cls.__tablename__ == 'library_dataset_collection_tag_association' with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -391,13 +391,18 @@ def test_LibraryDatasetCollectionTagAssociation(model, session, library_dataset_ obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id assert stored_obj.tag_id == tag.id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname assert stored_obj.value == value assert stored_obj.user_value == user_value + # test mapped relationships + assert stored_obj.dataset_collection.id == library_dataset_collection_association.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id def test_LibraryDatasetCollectionRatingAssociation( From 7722833a5dff8ba0fb198e5af5596914b9e27fca Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 15:38:31 -0400 Subject: [PATCH 062/225] Map ToolTagAssoc declaratively (see note) The `value` attr passed to ToolTagAssoc's constructor was overwritten with None: this cannot be right (you couldn't set it via __init__, yet it was a parameter). This is fixed now. --- lib/galaxy/model/__init__.py | 21 ++++++++++++--------- lib/galaxy/model/mapping.py | 18 +----------------- test/unit/model/test_model.py | 11 ++++++++--- 3 files changed, 21 insertions(+), 29 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 26765e004c2..ace01c01842 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6931,17 +6931,20 @@ class LibraryDatasetCollectionTagAssociation(Base, ItemTagAssociation, Represent user = relationship('User') -class ToolTagAssociation(ItemTagAssociation, RepresentById): - def __init__(self, id=None, user=None, tool_id=None, tag_id=None, user_tname=None, value=None): - self.id = id - self.user = user - self.tool_id = tool_id - self.tag_id = tag_id - self.user_tname = user_tname - self.value = None - self.user_value = None +class ToolTagAssociation(Base, ItemTagAssociation, RepresentById): + __tablename__ = 'tool_tag_association' + id = Column('id', Integer, primary_key=True) + tool_id = Column('tool_id', TrimmedString(255), index=True) + tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column('user_tname', TrimmedString(255), index=True) + value = Column('value', TrimmedString(255), index=True) + user_value = Column('user_value', TrimmedString(255), index=True) + tag = relationship('Tag', back_populates='tagged_tools') + user = relationship('User') + # Item annotation classes. class HistoryAnnotationAssociation(Base, RepresentById): __tablename__ = 'history_annotation_association' diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 1825b2613c8..8e1e85539db 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1115,16 +1115,6 @@ model.Tag.table = Table( Column("name", TrimmedString(255)), UniqueConstraint("name")) -model.ToolTagAssociation.table = Table( - "tool_tag_association", metadata, - Column("id", Integer, primary_key=True), - Column("tool_id", TrimmedString(255), index=True), - Column("tag_id", Integer, ForeignKey("tag.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("user_tname", TrimmedString(255), index=True), - Column("value", TrimmedString(255), index=True), - Column("user_value", TrimmedString(255), index=True)) - model.UserPreference.table = Table( "user_preference", metadata, Column("id", Integer, primary_key=True), @@ -2144,15 +2134,9 @@ mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( tagged_visualizations=relation(model.VisualizationTagAssociation, back_populates='tag'), tagged_history_dataset_collections=relation(model.HistoryDatasetCollectionTagAssociation, back_populates='tag'), tagged_library_dataset_collections=relation(model.LibraryDatasetCollectionTagAssociation, back_populates='tag'), + tagged_tools=relation(model.ToolTagAssociation, back_populates='tag'), )) - -def tag_mapping(tag_association_class, backref_name): - simple_mapping(tag_association_class, tag=relation(model.Tag, backref=backref_name), user=relation(model.User)) - - -tag_mapping(model.ToolTagAssociation, "tagged_tools") - mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( # user=relation( model.User.mapper ), user=relation(model.User), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index ac196c0dbb6..dfbd18034d3 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -702,7 +702,7 @@ def test_StoredWorkflowTagAssociation(model, session, stored_workflow, tag, user def test_ToolTagAssociation(model, session, tag, user): cls = model.ToolTagAssociation - # TODO assert cls.__tablename__ == '' + assert cls.__tablename__ == 'tool_tag_association' with dbcleanup(session, cls): user_tname, value, user_value, tool_id = 'a', 'b', 'c', 'd' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -711,13 +711,18 @@ def test_ToolTagAssociation(model, session, tag, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.tool_id == tool_id assert stored_obj.tag_id == tag.id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname + assert stored_obj.value == value assert stored_obj.user_value == user_value - assert stored_obj.value is None # TODO this is a confusing special case; improve? + # test mapped relationships + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id def testUserAction(model, session, user, galaxy_session): From 0f06c4bbf4a9d6784f84ea7f897d55880245db82 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 15:44:56 -0400 Subject: [PATCH 063/225] Fix test: rename attr tagged_workflows > tagged_stored_workflows (see 01b6bdba06757f300fde6af3718854a301b577e3) --- test/unit/data/test_galaxy_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py index 27b9b3a0a12..a5b347c9c55 100644 --- a/test/unit/data/test_galaxy_mapping.py +++ b/test/unit/data/test_galaxy_mapping.py @@ -253,7 +253,7 @@ class MappingTests(BaseModelTestCase): sw = model.StoredWorkflow() sw.user = u - tag_and_test(sw, model.StoredWorkflowTagAssociation, "tagged_workflows") + tag_and_test(sw, model.StoredWorkflowTagAssociation, "tagged_stored_workflows") h = model.History(name="History for Tagging", user=u) tag_and_test(h, model.HistoryTagAssociation, "tagged_histories") From a407a22437f93c2145e9697e89c556750694e707 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 24 Jun 2021 15:49:26 -0400 Subject: [PATCH 064/225] Remove constructor from ItemTagAssoc This is redundant and overwritten by Base --- lib/galaxy/model/__init__.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ace01c01842..26442be223e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6771,14 +6771,6 @@ class ItemTagAssociation(Dictifiable): super().__init_subclass__(**kwargs) cls.associated_item_names.append(cls.__name__.replace("TagAssociation", "")) - def __init__(self, user=None, item_id=None, tag_id=None, user_tname=None, value=None): - self.user = user - self.item_id = item_id - self.tag_id = tag_id - self.user_tname = user_tname - self.value = value - self.user_value = None - def copy(self, cls=None): if cls: new_ta = cls() @@ -6808,7 +6800,7 @@ class HistoryTagAssociation(Base, ItemTagAssociation, RepresentById): class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'history_dataset_association_tag_association' - + id = Column('id', Integer, primary_key=True) history_dataset_association_id = Column('history_dataset_association_id', Integer, ForeignKey('history_dataset_association.id'), index=True) @@ -6856,7 +6848,7 @@ class PageTagAssociation(Base, ItemTagAssociation, RepresentById): class WorkflowStepTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'workflow_step_tag_association' - + id = Column('id', Integer, primary_key=True) workflow_step_id = Column('workflow_step_id', Integer, ForeignKey('workflow_step.id'), index=True) tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) @@ -6944,7 +6936,7 @@ class ToolTagAssociation(Base, ItemTagAssociation, RepresentById): tag = relationship('Tag', back_populates='tagged_tools') user = relationship('User') - + # Item annotation classes. class HistoryAnnotationAssociation(Base, RepresentById): __tablename__ = 'history_annotation_association' From 7b5fcd6f4ff5367d3477facc816263c90527ff28 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 25 Jun 2021 17:08:34 -0400 Subject: [PATCH 065/225] Map Tag declaratively; first test for associations objects --- lib/galaxy/model/__init__.py | 32 +++++++-- lib/galaxy/model/mapping.py | 26 ------- test/unit/model/test_model.py | 124 +++++++++++++++++++++++++++++++++- 3 files changed, 149 insertions(+), 33 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 26442be223e..b76919fa95c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6751,12 +6751,32 @@ class TransferJob(RepresentById): self.params = params -class Tag(RepresentById): - def __init__(self, id=None, type=None, parent_id=None, name=None): - self.id = id - self.type = type - self.parent_id = parent_id - self.name = name +class Tag(Base, RepresentById): + __tablename__ = 'tag' + __table_args__ = ( + UniqueConstraint('name'), + ) + + id = Column('id', Integer, primary_key=True) + type = Column('type', Integer) + parent_id = Column('parent_id', Integer, ForeignKey('tag.id')) + name = Column('name', TrimmedString(255)) + children = relationship('Tag', back_populates='parent') + parent = relationship('Tag', back_populates='children', remote_side=[id]) + tagged_histories = relationship('HistoryTagAssociation', back_populates='tag') + tagged_history_dataset_associations = relationship( + 'HistoryDatasetAssociationTagAssociation', back_populates='tag') + tagged_library_dataset_dataset_associations = relationship( + 'LibraryDatasetDatasetAssociationTagAssociation', back_populates='tag') + tagged_pages = relationship('PageTagAssociation', back_populates='tag') + tagged_workflow_steps = relationship('WorkflowStepTagAssociation', back_populates='tag') + tagged_stored_workflows = relationship('StoredWorkflowTagAssociation', back_populates='tag') + tagged_visualizations = relationship('VisualizationTagAssociation', back_populates='tag') + tagged_history_dataset_collections = relationship( + 'HistoryDatasetCollectionTagAssociation', back_populates='tag') + tagged_library_dataset_collections = relationship( + 'LibraryDatasetCollectionTagAssociation', back_populates='tag') + tagged_tools = relationship('ToolTagAssociation', back_populates='tag') def __str__(self): return "Tag(id=%s, type=%i, parent_id=%s, name=%s)" % (self.id, self.type or -1, self.parent_id, self.name) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 8e1e85539db..1618c4c7b46 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -28,7 +28,6 @@ from sqlalchemy import ( Text, true, Unicode, - UniqueConstraint, ) from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.orderinglist import ordering_list @@ -1106,15 +1105,6 @@ model.DataManagerJobAssociation.table = Table( Index('ix_data_manager_job_association_data_manager_id', 'data_manager_id', mysql_length=200), ) -# Tagging tables. -model.Tag.table = Table( - "tag", metadata, - Column("id", Integer, primary_key=True), - Column("type", Integer), - Column("parent_id", Integer, ForeignKey("tag.id")), - Column("name", TrimmedString(255)), - UniqueConstraint("name")) - model.UserPreference.table = Table( "user_preference", metadata, Column("id", Integer, primary_key=True), @@ -2121,22 +2111,6 @@ mapper_registry.map_imperatively(model.VisualizationUserShareAssociation, model. backref='users_shared_with') )) -# Tag tables. - -mapper_registry.map_imperatively(model.Tag, model.Tag.table, properties=dict( - children=relation(model.Tag, backref=backref('parent', remote_side=[model.Tag.table.c.id])), - tagged_histories=relation(model.HistoryTagAssociation, back_populates='tag'), - tagged_history_dataset_associations=relation(model.HistoryDatasetAssociationTagAssociation, back_populates='tag'), - tagged_library_dataset_dataset_associations=relation(model.LibraryDatasetDatasetAssociationTagAssociation, back_populates='tag'), - tagged_pages=relation(model.PageTagAssociation, back_populates='tag'), - tagged_workflow_steps=relation(model.WorkflowStepTagAssociation, back_populates='tag'), - tagged_stored_workflows=relation(model.StoredWorkflowTagAssociation, back_populates='tag'), - tagged_visualizations=relation(model.VisualizationTagAssociation, back_populates='tag'), - tagged_history_dataset_collections=relation(model.HistoryDatasetCollectionTagAssociation, back_populates='tag'), - tagged_library_dataset_collections=relation(model.LibraryDatasetCollectionTagAssociation, back_populates='tag'), - tagged_tools=relation(model.ToolTagAssociation, back_populates='tag'), -)) - mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( # user=relation( model.User.mapper ), user=relation(model.User), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index dfbd18034d3..afb473dbb19 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -339,6 +339,7 @@ def test_HistoryRatingAssociation(model, session, history, user): def test_HistoryTagAssociation(model, session, history, tag, user): cls = model.HistoryTagAssociation assert cls.__tablename__ == 'history_tag_association' + with dbcleanup(session, cls): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) @@ -700,6 +701,68 @@ def test_StoredWorkflowTagAssociation(model, session, stored_workflow, tag, user assert stored_obj.user.id == user.id +def test_Tag( + model, + session, + tag, + history_tag_association, + history_dataset_association_tag_association, + library_dataset_dataset_association_tag_association, + page_tag_association, + workflow_step_tag_association, + stored_workflow_tag_association, + visualization_tag_association, + history_dataset_collection_tag_association, + library_dataset_collection_tag_association, + tool_tag_association, +): + cls = model.Tag + assert cls.__tablename__ == 'tag' + assert has_unique_constraint(cls.__table__, ('name',)) + + with dbcleanup(session, cls): + type_, name = 1, 'a' + child_tag = cls() + obj = cls(type=type_, parent_id=tag.id, name=name) + obj.children.append(child_tag) + + def add_association(assoc_object, assoc_attribute): + assoc_object.tag = tag + getattr(obj, assoc_attribute).append(assoc_object) + + add_association(history_tag_association, 'tagged_histories') + add_association(history_dataset_association_tag_association, 'tagged_history_dataset_associations') + add_association(library_dataset_dataset_association_tag_association, 'tagged_library_dataset_dataset_associations') + add_association(page_tag_association, 'tagged_pages') + add_association(workflow_step_tag_association, 'tagged_workflow_steps') + add_association(stored_workflow_tag_association, 'tagged_stored_workflows') + add_association(visualization_tag_association, 'tagged_visualizations') + add_association(history_dataset_collection_tag_association, 'tagged_history_dataset_collections') + add_association(library_dataset_collection_tag_association, 'tagged_library_dataset_collections') + add_association(tool_tag_association, 'tagged_tools') + + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test fields + assert stored_obj.id == obj_id + assert stored_obj.type == type_ + assert stored_obj.name == name + # test relationships + assert stored_obj.parent.id == tag.id + assert stored_obj.children == [child_tag] + assert stored_obj.tagged_histories == [history_tag_association] + assert stored_obj.tagged_history_dataset_associations == [history_dataset_association_tag_association] + assert stored_obj.tagged_library_dataset_dataset_associations == [library_dataset_dataset_association_tag_association] + assert stored_obj.tagged_pages == [page_tag_association] + assert stored_obj.tagged_workflow_steps == [workflow_step_tag_association] + assert stored_obj.tagged_stored_workflows == [stored_workflow_tag_association] + assert stored_obj.tagged_visualizations == [visualization_tag_association] + assert stored_obj.tagged_history_dataset_collections == [history_dataset_collection_tag_association] + assert stored_obj.tagged_library_dataset_collections == [library_dataset_collection_tag_association] + assert stored_obj.tagged_tools == [tool_tag_association] + + def test_ToolTagAssociation(model, session, tag, user): cls = model.ToolTagAssociation assert cls.__tablename__ == 'tool_tag_association' @@ -711,7 +774,6 @@ def test_ToolTagAssociation(model, session, tag, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns assert stored_obj.id == obj_id assert stored_obj.tool_id == tool_id @@ -997,12 +1059,30 @@ def history_dataset_association(model, session, dataset): yield from dbcleanup_wrapper(session, hda) +@pytest.fixture +def history_dataset_association_tag_association(model, session): + hdata = model.HistoryDatasetAssociationTagAssociation() + yield from dbcleanup_wrapper(session, hdata) + + @pytest.fixture def history_dataset_collection_association(model, session): hdca = model.HistoryDatasetCollectionAssociation() yield from dbcleanup_wrapper(session, hdca) +@pytest.fixture +def history_dataset_collection_tag_association(model, session): + hdcta = model.HistoryDatasetCollectionTagAssociation() + yield from dbcleanup_wrapper(session, hdcta) + + +@pytest.fixture +def history_tag_association(model, session): + hta = model.HistoryTagAssociation() + yield from dbcleanup_wrapper(session, hta) + + @pytest.fixture def galaxy_session(model, session, user): s = model.GalaxySession() @@ -1021,12 +1101,24 @@ def library_dataset_collection_association(model, session): yield from dbcleanup_wrapper(session, ldca) +@pytest.fixture +def library_dataset_collection_tag_association(model, session): + ldcta = model.LibraryDatasetCollectionTagAssociation() + yield from dbcleanup_wrapper(session, ldcta) + + @pytest.fixture def library_dataset_dataset_association(model, session): ldda = model.LibraryDatasetDatasetAssociation() yield from dbcleanup_wrapper(session, ldda) +@pytest.fixture +def library_dataset_dataset_association_tag_association(model, session): + lddata = model.LibraryDatasetDatasetAssociationTagAssociation() + yield from dbcleanup_wrapper(session, lddata) + + @pytest.fixture def page(model, session, user): p = model.Page() @@ -1034,6 +1126,12 @@ def page(model, session, user): yield from dbcleanup_wrapper(session, p) +@pytest.fixture +def page_tag_association(model, session): + pta = model.PageTagAssociation() + yield from dbcleanup_wrapper(session, pta) + + @pytest.fixture def quota(model, session): q = model.Quota(get_random_string(), 'b') @@ -1053,12 +1151,24 @@ def stored_workflow(model, session, user): yield from dbcleanup_wrapper(session, w) +@pytest.fixture +def stored_workflow_tag_association(model, session): + swta = model.StoredWorkflowTagAssociation() + yield from dbcleanup_wrapper(session, swta) + + @pytest.fixture def tag(model, session): t = model.Tag() yield from dbcleanup_wrapper(session, t) +@pytest.fixture +def tool_tag_association(model, session): + tta = model.ToolTagAssociation() + yield from dbcleanup_wrapper(session, tta) + + @pytest.fixture def user(model, session): u = model.User(email='test@example.com', password='password') @@ -1078,6 +1188,12 @@ def visualization(model, session, user): yield from dbcleanup_wrapper(session, v) +@pytest.fixture +def visualization_tag_association(model, session): + vta = model.VisualizationTagAssociation() + yield from dbcleanup_wrapper(session, vta) + + @pytest.fixture def workflow(model, session): w = model.Workflow() @@ -1091,6 +1207,12 @@ def workflow_step(model, session, workflow): yield from dbcleanup_wrapper(session, s) +@pytest.fixture +def workflow_step_tag_association(model, session): + wsta = model.WorkflowStepTagAssociation() + yield from dbcleanup_wrapper(session, wsta) + + @contextmanager def dbcleanup(session, cls): """ From d7a97053dd1060cfc90f83a0073c4af260f1ceb3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 25 Jun 2021 17:33:42 -0400 Subject: [PATCH 066/225] Refactor APIKeys, model test - Remove redundant constructor - Test all fields and relationships - Ensure unique constraint --- lib/galaxy/model/__init__.py | 4 ---- test/unit/model/test_model.py | 7 +++++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b76919fa95c..2d81d12d978 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -7236,10 +7236,6 @@ class APIKeys(Base, RepresentById): key = Column('key', TrimmedString(32), index=True, unique=True) user = relationship('User', back_populates='api_keys') - def __init__(self, user_id=None, key=None): - self.user_id = user_id - self.key = key - def copy_list(lst, *args, **kwds): if lst is None: diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index afb473dbb19..a7fb09d96d6 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -16,14 +16,17 @@ def testAPIKeys(model, session, user): cls = model.APIKeys assert cls.__tablename__ == 'api_keys' with dbcleanup(session, cls): - user_id, key = user.id, 'a' - obj = cls(user_id, key) + create_time, user_id, key = datetime.now(), user.id, get_random_string() + obj = cls(user_id=user_id, key=key, create_time=create_time) obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time assert stored_obj.user_id == user_id assert stored_obj.key == key + # test mapped relationships assert stored_obj.user.id == user.id From f8390c59d95f75a43b9a892cd87600545f1af442 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 25 Jun 2021 18:01:24 -0400 Subject: [PATCH 067/225] Clean up tests, fixtures - Ensure all mapped fields and all mapped relationships are tested explicitly - Visually separate testing mapped fields from testing mapped relationships - Reformat for consistency, readability --- test/unit/model/test_model.py | 417 +++++++++++++++++++++++++++------- 1 file changed, 329 insertions(+), 88 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index a7fb09d96d6..9fef0bdfa35 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1,6 +1,6 @@ import random from contextlib import contextmanager -from datetime import datetime +from datetime import datetime, timedelta import pytest from sqlalchemy import ( @@ -39,16 +39,18 @@ def test_CloudAuthz(model, session, user, user_authnz_token): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.user_id == user.id assert stored_obj.provider == provider assert stored_obj.config == config assert stored_obj.authn_id == user_authnz_token.id - assert stored_obj.description == description assert stored_obj.tokens is None assert stored_obj.last_update assert stored_obj.last_activity + assert stored_obj.description == description assert stored_obj.create_time + # test mapped relationships assert stored_obj.user.id == user.id assert stored_obj.authn.id == user_authnz_token.id @@ -71,7 +73,9 @@ def test_CustosAuthnzToken(model, session, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id + assert stored_obj.user_id == user.id assert stored_obj.external_user_id == external_user_id assert stored_obj.provider == provider assert stored_obj.access_token == access_token @@ -79,7 +83,8 @@ def test_CustosAuthnzToken(model, session, user): assert stored_obj.refresh_token == refresh_token assert stored_obj.expiration_time == expiration_time assert stored_obj.refresh_expiration_time == refresh_expiration_time - assert stored_obj.user == user + # test mapped relationships + assert stored_obj.user.id == user.id def test_DatasetPermissions(model, session, dataset, role): @@ -88,11 +93,21 @@ def test_DatasetPermissions(model, session, dataset, role): with dbcleanup(session, cls): action = 'a' obj = cls(action, dataset, role) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time assert stored_obj.action == action + assert stored_obj.dataset_id == dataset.id + assert stored_obj.role_id == role.id + # test mapped relationships assert stored_obj.dataset == dataset assert stored_obj.role == role @@ -103,12 +118,21 @@ def test_DefaultQuotaAssociation(model, session, quota): with dbcleanup(session, cls): type = cls.types.REGISTERED obj = cls(type, quota) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time assert stored_obj.type == type - assert stored_obj.quota == quota + assert stored_obj.quota_id == quota.id + # test mapped relationships + assert stored_obj.quota.id == quota.id def test_DefaultUserPermissions(model, session, user, role): @@ -120,10 +144,14 @@ def test_DefaultUserPermissions(model, session, user, role): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.user == user + assert stored_obj.user_id == user.id assert stored_obj.action == action - assert stored_obj.role == role + assert stored_obj.role_id == role.id + # test mapped relationships + assert stored_obj.user.id == user.id + assert stored_obj.role.id == role.id def test_DynamicTool(model, session, workflow_step): @@ -141,20 +169,28 @@ def test_DynamicTool(model, session, workflow_step): value = 'f' obj = cls(tool_format, tool_id, tool_version, tool_path, tool_directory, uuid, active, hidden, value) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time obj.workflow_steps.append(workflow_step) obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.tool_format == tool_format + assert stored_obj.uuid + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time assert stored_obj.tool_id == tool_id assert stored_obj.tool_version == tool_version + assert stored_obj.tool_format == tool_format assert stored_obj.tool_path == tool_path assert stored_obj.tool_directory == tool_directory - assert stored_obj.uuid - assert stored_obj.active == active assert stored_obj.hidden == hidden + assert stored_obj.active == active assert stored_obj.value == value + # test mapped relationships assert workflow_step in stored_obj.workflow_steps @@ -163,12 +199,22 @@ def test_GroupQuotaAssociation(model, session, group, quota): assert cls.__tablename__ == 'group_quota_association' with dbcleanup(session, cls): obj = cls(group, quota) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.group == group - assert stored_obj.quota == quota + assert stored_obj.group_id == group.id + assert stored_obj.quota_id == quota.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + # test mapped relationships + assert stored_obj.group.id == group.id + assert stored_obj.quota.id == quota.id def test_GroupRoleAssociation(model, session, group, role): @@ -176,12 +222,22 @@ def test_GroupRoleAssociation(model, session, group, role): assert cls.__tablename__ == 'group_role_association' with dbcleanup(session, cls): obj = cls(group, role) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.group == group - assert stored_obj.role == role + assert stored_obj.group_id == group.id + assert stored_obj.role_id == role.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + # test mapped relationships + assert stored_obj.group.id == group.id + assert stored_obj.role.id == role.id def test_HistoryAnnotationAssociation(model, session, history, user): @@ -197,10 +253,14 @@ def test_HistoryAnnotationAssociation(model, session, history, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.history == history - assert stored_obj.user == user + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation + # test mapped relationships + assert stored_obj.history.id == history.id + assert stored_obj.user.id == user.id def test_HistoryDatasetAssociationAnnotationAssociation( @@ -217,10 +277,14 @@ def test_HistoryDatasetAssociationAnnotationAssociation( obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.hda == history_dataset_association - assert stored_obj.user == user + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation + # test mapped relationships + assert stored_obj.hda.id == history_dataset_association.id + assert stored_obj.user.id == user.id def test_HistoryDatasetAssociationRatingAssociation(model, session, history_dataset_association, user): @@ -232,10 +296,14 @@ def test_HistoryDatasetAssociationRatingAssociation(model, session, history_data obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.user == user - assert stored_obj.history_dataset_association == history_dataset_association + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.user_id == user.id assert stored_obj.rating == rating + # test mapped relationships + assert stored_obj.history_dataset_association.id == history_dataset_association.id + assert stored_obj.user.id == user.id def test_HistoryDatasetAssociationTagAssociation( @@ -277,10 +345,14 @@ def test_HistoryDatasetCollectionAssociationAnnotationAssociation( obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.history_dataset_collection == history_dataset_collection_association - assert stored_obj.user == user + assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation + # test mapped relationships + assert stored_obj.history_dataset_collection.id == history_dataset_collection_association.id + assert stored_obj.user.id == user.id def test_HistoryDatasetCollectionRatingAssociation( @@ -293,10 +365,14 @@ def test_HistoryDatasetCollectionRatingAssociation( obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.user == user - assert stored_obj.dataset_collection == history_dataset_collection_association + assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.user.id == user.id assert stored_obj.rating == rating + # test mapped relationships + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id + assert stored_obj.user.id == user.id def test_HistoryDatasetCollectionTagAssociation(model, session, history_dataset_collection_association, tag, user): @@ -333,10 +409,14 @@ def test_HistoryRatingAssociation(model, session, history, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.user == user - assert stored_obj.history == history + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id assert stored_obj.rating == rating + # test mapped relationships + assert stored_obj.history.id == history.id + assert stored_obj.user.id == user.id def test_HistoryTagAssociation(model, session, history, tag, user): @@ -378,10 +458,14 @@ def test_LibraryDatasetCollectionAnnotationAssociation( obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.dataset_collection == library_dataset_collection_association - assert stored_obj.user == user + assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id + assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation + # test mapped relationships + assert stored_obj.dataset_collection.id == library_dataset_collection_association.id + assert stored_obj.user.id == user.id def test_LibraryDatasetCollectionTagAssociation(model, session, library_dataset_collection_association, tag, user): @@ -419,10 +503,14 @@ def test_LibraryDatasetCollectionRatingAssociation( obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.user == user - assert stored_obj.dataset_collection == library_dataset_collection_association + assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id + assert stored_obj.user_id == user.id assert stored_obj.rating == rating + # test mapped relationships + assert stored_obj.dataset_collection.id == library_dataset_collection_association.id + assert stored_obj.user.id == user.id def test_LibraryDatasetDatasetAssociationTagAssociation( @@ -464,10 +552,14 @@ def test_PageAnnotationAssociation(model, session, page, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.page == page - assert stored_obj.user == user + assert stored_obj.page_id == page.id + assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation + # test mapped relationships + assert stored_obj.page.id == page.id + assert stored_obj.user.id == user.id def test_PageRatingAssociation(model, session, page, user): @@ -479,10 +571,14 @@ def test_PageRatingAssociation(model, session, page, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.user == user - assert stored_obj.page == page + assert stored_obj.page_id == page.id + assert stored_obj.user_id == user.id assert stored_obj.rating == rating + # test mapped relationships + assert stored_obj.page.id == page.id + assert stored_obj.user.id == user.id def test_PageRevision(model, session, page): @@ -491,17 +587,24 @@ def test_PageRevision(model, session, page): with dbcleanup(session, cls): obj = cls() obj.page = page + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + title, content = 'a', 'b' + obj.title = title + obj.content = content obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.create_time - assert stored_obj.update_time + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time assert stored_obj.page_id == page.id - assert stored_obj.title is None - assert stored_obj.content is None + assert stored_obj.title == title + assert stored_obj.content == content assert stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT - assert stored_obj.page.id == page.id def test_PageTagAssociation(model, session, page, tag, user): @@ -539,9 +642,13 @@ def test_PageUserShareAssociation(model, session, page, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.page == page - assert stored_obj.user == user + assert stored_obj.page_id == page.id + assert stored_obj.user_id == user.id + # test mapped relationships + assert stored_obj.page.id == page.id + assert stored_obj.user.id == user.id def test_PSAAssociation(model, session): @@ -553,6 +660,7 @@ def test_PSAAssociation(model, session): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.server_url == server_url assert stored_obj.handle == handle @@ -572,6 +680,7 @@ def test_PSACode(model, session): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.email == email assert stored_obj.code == code @@ -586,6 +695,7 @@ def test_PSANonce(model, session): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.server_url assert stored_obj.timestamp == timestamp @@ -601,6 +711,7 @@ def test_PSAPartial(model, session): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.token == token assert stored_obj.data == data @@ -608,41 +719,75 @@ def test_PSAPartial(model, session): assert stored_obj.backend == backend -def test_Quota(model, session): +def test_Quota( + model, + session, + default_quota_association, + group_quota_association, + user_quota_association +): cls = model.Quota assert cls.__tablename__ == 'quota' with dbcleanup(session, cls): - name, description = get_random_string(), 'b' - obj = cls(name, description) + name, description, amount, operation = get_random_string(), 'b', 42, '+' + obj = cls(name, description, amount, operation) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + + def add_association(assoc_object, assoc_attribute): + assoc_object.quota = obj + getattr(obj, assoc_attribute).append(assoc_object) + + add_association(default_quota_association, 'default') + add_association(group_quota_association, 'groups') + add_association(user_quota_association, 'users') + obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.create_time - assert stored_obj.update_time + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time assert stored_obj.name == name assert stored_obj.description == description - assert stored_obj.bytes == 0 - assert stored_obj.operation == '=' + assert stored_obj.bytes == amount + assert stored_obj.operation == operation assert stored_obj.deleted is False + # test mapped relationships + assert stored_obj.default == [default_quota_association] + assert stored_obj.groups == [group_quota_association] + assert stored_obj.users == [user_quota_association] -def test_Role(model, session): +def test_Role(model, session, dataset_permission, group_role_association): cls = model.Role assert cls.__tablename__ == 'role' with dbcleanup(session, cls): - name, description = get_random_string(), 'b' - obj = cls(name, description) + name, description, type_, deleted = get_random_string(), 'b', cls.types.SYSTEM, True + obj = cls(name, description, type_, deleted) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + obj.dataset_actions.append(dataset_permission) + obj.groups.append(group_role_association) obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.create_time - assert stored_obj.update_time + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time assert stored_obj.name == name assert stored_obj.description == description - assert stored_obj.type == model.Role.types.SYSTEM - assert stored_obj.deleted is False + assert stored_obj.type == type_ + assert stored_obj.deleted == deleted + # test mapped relationships + assert stored_obj.dataset_actions == [dataset_permission] + assert stored_obj.groups == [group_role_association] def test_StoredWorkflowAnnotationAssociation(model, session, stored_workflow, user): @@ -652,16 +797,20 @@ def test_StoredWorkflowAnnotationAssociation(model, session, stored_workflow, us with dbcleanup(session, cls): annotation = 'a' obj = cls() - obj.user = user obj.stored_workflow = stored_workflow + obj.user = user obj.annotation = annotation obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.stored_workflow == stored_workflow - assert stored_obj.user == user + assert stored_obj.stored_workflow_id == stored_workflow.id + assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation + # test mapped relationships + assert stored_obj.stored_workflow.id == stored_workflow.id + assert stored_obj.user.id == user.id def test_StoredWorkflowRatingAssociation(model, session, stored_workflow, user): @@ -673,10 +822,14 @@ def test_StoredWorkflowRatingAssociation(model, session, stored_workflow, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.user == user - assert stored_obj.stored_workflow == stored_workflow + assert stored_obj.stored_workflow_id == stored_workflow.id + assert stored_obj.user_id == user.id assert stored_obj.rating == rating + # test mapped relationships + assert stored_obj.stored_workflow.id == stored_workflow.id + assert stored_obj.user.id == user.id def test_StoredWorkflowTagAssociation(model, session, stored_workflow, tag, user): @@ -707,7 +860,6 @@ def test_StoredWorkflowTagAssociation(model, session, stored_workflow, tag, user def test_Tag( model, session, - tag, history_tag_association, history_dataset_association_tag_association, library_dataset_dataset_association_tag_association, @@ -724,13 +876,15 @@ def test_Tag( assert has_unique_constraint(cls.__table__, ('name',)) with dbcleanup(session, cls): - type_, name = 1, 'a' + parent_tag = cls() child_tag = cls() - obj = cls(type=type_, parent_id=tag.id, name=name) + type_, name = 1, 'a' + obj = cls(type=type_, name=name) + obj.parent = parent_tag obj.children.append(child_tag) def add_association(assoc_object, assoc_attribute): - assoc_object.tag = tag + assoc_object.tag = obj getattr(obj, assoc_attribute).append(assoc_object) add_association(history_tag_association, 'tagged_histories') @@ -747,12 +901,13 @@ def test_Tag( obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) - # test fields + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.type == type_ + assert stored_obj.parent_id == parent_tag.id assert stored_obj.name == name - # test relationships - assert stored_obj.parent.id == tag.id + # test mapped relationships + assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] assert stored_obj.tagged_histories == [history_tag_association] assert stored_obj.tagged_history_dataset_associations == [history_dataset_association_tag_association] @@ -796,16 +951,20 @@ def testUserAction(model, session, user, galaxy_session): with dbcleanup(session, cls): action, params, context = 'a', 'b', 'c' obj = cls(user, galaxy_session.id, action, params, context) + create_time = datetime.now() + obj.create_time = create_time obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.create_time + assert stored_obj.create_time == create_time assert stored_obj.user_id == user.id assert stored_obj.session_id == galaxy_session.id assert stored_obj.action == action assert stored_obj.context == context assert stored_obj.params == params + # test mapped relationships assert stored_obj.user.id == user.id @@ -813,15 +972,22 @@ def test_UserAddress(model, session, user): cls = model.UserAddress assert cls.__tablename__ == 'user_address' with dbcleanup(session, cls): - desc, name, institution, address, city, state, postal_code, country, phone = \ - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i' + desc, name, institution, address, city, state, postal_code, country, phone, deleted, purged = \ + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', True, False obj = cls(user, desc, name, institution, address, city, state, postal_code, country, phone) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + obj.deleted = deleted + obj.purged = purged obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.create_time - assert stored_obj.update_time + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time assert stored_obj.user_id == user.id assert stored_obj.desc == desc assert stored_obj.name == name @@ -832,9 +998,10 @@ def test_UserAddress(model, session, user): assert stored_obj.postal_code == postal_code assert stored_obj.country == country assert stored_obj.phone == phone - assert stored_obj.deleted is False - assert stored_obj.purged is False - assert stored_obj.user == user + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + # test mapped relationships + assert stored_obj.user.id == user.id def test_UserAuthnzToken(model, session, user, cloud_authz): @@ -848,6 +1015,7 @@ def test_UserAuthnzToken(model, session, user, cloud_authz): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.user_id == user.id assert stored_obj.uid == uid @@ -855,8 +1023,9 @@ def test_UserAuthnzToken(model, session, user, cloud_authz): assert stored_obj.extra_data == extra_data assert stored_obj.lifetime == lifetime assert stored_obj.assoc_type == assoc_type + # test mapped relationships + assert stored_obj.cloudauthz == [cloud_authz] assert stored_obj.user.id == user.id - assert cloud_authz in stored_obj.cloudauthz def test_UserGroupAssociation(model, session, user, group): @@ -864,12 +1033,22 @@ def test_UserGroupAssociation(model, session, user, group): assert cls.__tablename__ == 'user_group_association' with dbcleanup(session, cls): obj = cls(user, group) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.user == user - assert stored_obj.group == group + assert stored_obj.user_id == user.id + assert stored_obj.group_id == group.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + # test mapped relationships + assert stored_obj.user.id == user.id + assert stored_obj.group.id == group.id def test_UserQuotaAssociation(model, session, user, quota): @@ -877,12 +1056,22 @@ def test_UserQuotaAssociation(model, session, user, quota): assert cls.__tablename__ == 'user_quota_association' with dbcleanup(session, cls): obj = cls(user, quota) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.user == user - assert stored_obj.quota == quota + assert stored_obj.user_id == user.id + assert stored_obj.quota_id == quota.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + # test mapped relationships + assert stored_obj.user.id == user.id + assert stored_obj.quota.id == quota.id def test_VisualizationAnnotationAssociation(model, session, visualization, user): @@ -898,10 +1087,14 @@ def test_VisualizationAnnotationAssociation(model, session, visualization, user) obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.visualization == visualization - assert stored_obj.user == user + assert stored_obj.visualization_id == visualization.id + assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation + # test mapped relationships + assert stored_obj.visualization.id == visualization.id + assert stored_obj.user.id == user.id def test_VisualizationRatingAssociation(model, session, visualization, user): @@ -913,10 +1106,14 @@ def test_VisualizationRatingAssociation(model, session, visualization, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.user == user - assert stored_obj.visualization == visualization + assert stored_obj.visualization_id == visualization.id + assert stored_obj.user_id == user.id assert stored_obj.rating == rating + # test mapped relationships + assert stored_obj.visualization.id == visualization.id + assert stored_obj.user.id == user.id def test_VisualizationRevision(model, session, visualization): @@ -926,16 +1123,22 @@ def test_VisualizationRevision(model, session, visualization): with dbcleanup(session, cls): visualization, title, dbkey, config = visualization, 'a', 'b', 'c' obj = cls(visualization, title, dbkey, config) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.create_time - assert stored_obj.update_time + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time assert stored_obj.visualization_id == visualization.id assert stored_obj.title == title assert stored_obj.dbkey == dbkey assert stored_obj.config == config + # test mapped relationships assert stored_obj.visualization.id == visualization.id @@ -946,14 +1149,17 @@ def test_WorkerProcess(model, session): with dbcleanup(session, cls): server_name, hostname = get_random_string(), 'a' obj = cls(server_name, hostname) + update_time = datetime.now() + obj.update_time = update_time obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id assert stored_obj.server_name == server_name assert stored_obj.hostname == hostname assert stored_obj.pid is None - assert stored_obj.update_time + assert stored_obj.update_time == update_time def test_WorkflowStepAnnotationAssociation(model, session, workflow_step, user): @@ -969,10 +1175,14 @@ def test_WorkflowStepAnnotationAssociation(model, session, workflow_step, user): obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns assert stored_obj.id == obj_id - assert stored_obj.workflow_step == workflow_step - assert stored_obj.user == user + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation + # test mapped relationships + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.user.id == user.id def test_WorkflowStepTagAssociation(model, session, workflow_step, tag, user): @@ -1050,6 +1260,31 @@ def dataset(model, session): yield from dbcleanup_wrapper(session, d) +@pytest.fixture +def dataset_permission(model, session, dataset): + d = model.DatasetPermissions('a', dataset) + yield from dbcleanup_wrapper(session, d) + + +@pytest.fixture +def default_quota_association(model, session, quota): + type_ = model.DefaultQuotaAssociation.types.REGISTERED + dqa = model.DefaultQuotaAssociation(type_, quota) + yield from dbcleanup_wrapper(session, dqa) + + +@pytest.fixture +def group_role_association(model, session): + gra = model.GroupRoleAssociation(None, None) + yield from dbcleanup_wrapper(session, gra) + + +@pytest.fixture +def group_quota_association(model, session): + gqa = model.GroupQuotaAssociation(None, None) + yield from dbcleanup_wrapper(session, gqa) + + @pytest.fixture def history(model, session): h = model.History() @@ -1184,6 +1419,12 @@ def user_authnz_token(model, session, user): yield from dbcleanup_wrapper(session, t) +@pytest.fixture +def user_quota_association(model, session): + uqa = model.UserQuotaAssociation(None, None) + yield from dbcleanup_wrapper(session, uqa) + + @pytest.fixture def visualization(model, session, user): v = model.Visualization() From ca6221eaf1a43452ae6cba1f71f91ef3fc6ee39e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 28 Jun 2021 20:32:58 -0400 Subject: [PATCH 068/225] Place Page model mapping under test (see note) Splitting this into 2 commits to record the process of ensuring mapping is correct on a sufficiently nontrivial model: Commit 1: Cover w/test; do not change model or mapping. Commit 2: Change model and mapping; do not change test. --- test/unit/model/test_model.py | 86 ++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 9fef0bdfa35..945da5b1f70 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -539,6 +539,65 @@ def test_LibraryDatasetDatasetAssociationTagAssociation( assert stored_obj.user.id == user.id +def test_Page( + model, + session, + user, + page_revision, + page_tag_association, + page_annotation_association, + page_rating_association, + page_user_share_association, +): + cls = model.Page + #assert cls.__tablename__ == 'page_annotation_association' + assert has_index(cls.__table__, ('slug',)) + with dbcleanup(session, cls): + title, deleted, importable, slug, published = 'a', True, True, 'b', True + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj = cls() + obj.user = user + obj.title = title + obj.deleted = deleted + obj.importable = importable + obj.slug = slug + obj.published = published + obj.create_time = create_time + obj.update_time = update_time + # This is OK for this test; however, page_revision.page != obj. Can we do better? + obj.latest_revision = page_revision + obj.revisions.append(page_revision) + obj.tags.append(page_tag_association) + obj.annotations.append(page_annotation_association) + obj.ratings.append(page_rating_association) + obj.users_shared_with.append(page_user_share_association) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.user_id == user.id + assert stored_obj.latest_revision_id == page_revision.id + assert stored_obj.title == title + assert stored_obj.deleted == deleted + assert stored_obj.importable == importable + assert stored_obj.slug == slug + assert stored_obj.published == published + # This doesn't test the average; need to use multiple ratings for that. + assert stored_obj.average_rating == page_rating_association.rating + # test mapped relationships + assert stored_obj.user.id == user.id + assert stored_obj.revisions == [page_revision] + assert stored_obj.latest_revision.id == page_revision.id + assert stored_obj.tags == [page_tag_association] + assert stored_obj.annotations == [page_annotation_association] + assert stored_obj.ratings == [page_rating_association] + assert stored_obj.users_shared_with == [page_user_share_association] + + def test_PageAnnotationAssociation(model, session, page, user): cls = model.PageAnnotationAssociation assert cls.__tablename__ == 'page_annotation_association' @@ -945,7 +1004,7 @@ def test_ToolTagAssociation(model, session, tag, user): assert stored_obj.user.id == user.id -def testUserAction(model, session, user, galaxy_session): +def test_UserAction(model, session, user, galaxy_session): cls = model.UserAction assert cls.__tablename__ == 'user_action' with dbcleanup(session, cls): @@ -1364,12 +1423,37 @@ def page(model, session, user): yield from dbcleanup_wrapper(session, p) +@pytest.fixture +def page_revision(model, session, page): + pr = model.PageRevision() + pr.page = page + yield from dbcleanup_wrapper(session, pr) + + +@pytest.fixture +def page_annotation_association(model, session): + paa = model.PageAnnotationAssociation() + yield from dbcleanup_wrapper(session, paa) + + +@pytest.fixture +def page_rating_association(model, session): + pra = model.PageRatingAssociation(None, None) + yield from dbcleanup_wrapper(session, pra) + + @pytest.fixture def page_tag_association(model, session): pta = model.PageTagAssociation() yield from dbcleanup_wrapper(session, pta) +@pytest.fixture +def page_user_share_association(model, session): + pra = model.PageUserShareAssociation() + yield from dbcleanup_wrapper(session, pra) + + @pytest.fixture def quota(model, session): q = model.Quota(get_random_string(), 'b') From d5c63b8019c70e091f0de222193f4bc2221cfa6f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 29 Jun 2021 02:17:12 -0400 Subject: [PATCH 069/225] Map Page declaratively (see note) - Add bidirectional `user` relationship to Page class. - Define avg column_property after all models have been mapped. - Add test to test calculation of average rating via column_property. --- lib/galaxy/model/__init__.py | 61 +++++++++++++++++++++++++++++++++-- lib/galaxy/model/mapping.py | 44 +------------------------ test/unit/model/test_model.py | 21 ++++++++++-- 3 files changed, 78 insertions(+), 48 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 2d81d12d978..73e6c678ce0 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -62,6 +62,7 @@ from sqlalchemy.exc import OperationalError from sqlalchemy.ext import hybrid from sqlalchemy.orm import ( aliased, + column_property, joinedload, object_session, Query, @@ -6586,8 +6587,52 @@ class CloudAuthz(Base, _HasTable): and self.config[k] == config[k]}) == len(self.config)) -class Page(Dictifiable, RepresentById): - # username needed for slug generation +class Page(Base, Dictifiable, RepresentById): + __tablename__ = 'page' + __table_args__ = ( + Index('ix_page_slug', 'slug', mysql_length=200), + ) + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True, nullable=False) + latest_revision_id = Column('latest_revision_id', Integer, + ForeignKey('page_revision.id', use_alter=True, name='page_latest_revision_id_fk'), index=True) + title = Column('title', TEXT) + deleted = Column('deleted', Boolean, index=True, default=False) + importable = Column('importable', Boolean, index=True, default=False) + slug = Column('slug', TEXT) + published = Column('published', Boolean, index=True, default=False) + user = relationship('User', back_populates='pages') + revisions = relationship( + 'PageRevision', + cascade="all, delete-orphan", + primaryjoin=('Page.id == PageRevision.page_id'), + back_populates='page') + latest_revision = relationship( + 'PageRevision', + post_update=True, + primaryjoin=('Page.latest_revision_id == PageRevision.id'), + lazy=False) + tags = relationship( + 'PageTagAssociation', + order_by='PageTagAssociation.id', + back_populates='page') + annotations = relationship( + 'PageAnnotationAssociation', + order_by='PageAnnotationAssociation.id', + back_populates='page') + ratings = relationship( + 'PageRatingAssociation', + order_by='PageRatingAssociation.id', + back_populates='page') + users_shared_with = relationship( + 'PageUserShareAssociation', + back_populates='page') + # `average_rating` added as column_property at the bottom of this module + # (cannot be added before PageRatingAssociation is defined) + dict_element_visible_keys = ['id', 'title', 'latest_revision_id', 'slug', 'published', 'importable', 'deleted', 'username'] def __init__(self): @@ -6608,6 +6653,7 @@ class Page(Dictifiable, RepresentById): rval['revision_ids'] = rev return rval + # username needed for slug generation @property def username(self): return self.user.username @@ -6623,7 +6669,9 @@ class PageRevision(Base, Dictifiable, RepresentById): title = Column('title', TEXT) content = Column('content', TEXT) content_format = Column('content_format', TrimmedString(32)) - + page = relationship( + 'Page', + primaryjoin=('Page.id == PageRevision.page_id')) DEFAULT_CONTENT_FORMAT = 'html' dict_element_visible_keys = ['id', 'page_id', 'title', 'content', 'content_format'] @@ -7258,3 +7306,10 @@ def _prepare_metadata_for_serialization(id_encoder, serialization_options, metad processed_metadata[name] = value return processed_metadata + + +# This must be defined after PageRatingAssociation has been mapped +Page.average_rating = column_property( + select(func.avg(PageRatingAssociation.rating)).where(PageRatingAssociation.page_id == Page.id).scalar_subquery(), + deferred=True +) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 1618c4c7b46..5e5947aadc5 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1045,22 +1045,6 @@ model.FormValues.table = Table( Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True), Column("content", MutableJSONType)) -model.Page.table = Table( - "page", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True, nullable=False), - Column("latest_revision_id", Integer, - ForeignKey("page_revision.id", use_alter=True, name='page_latest_revision_id_fk'), index=True), - Column("title", TEXT), - Column("deleted", Boolean, index=True, default=False), - Column("importable", Boolean, index=True, default=False), - Column("slug", TEXT), - Column("published", Boolean, index=True, default=False), - Index('ix_page_slug', 'slug', mysql_length=200), -) - model.Visualization.table = Table( "visualization", metadata, Column("id", Integer, primary_key=True), @@ -1408,6 +1392,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( api_keys=relation(model.APIKeys, back_populates="user", order_by=desc(model.APIKeys.create_time)), + pages=relation(model.Page, back_populates='user'), )) mapper_registry.map_imperatively(model.PasswordResetToken, model.PasswordResetToken.table, @@ -2042,33 +2027,6 @@ simple_mapping( dataset_collection=relation(model.HistoryDatasetCollectionAssociation), ) - -mapper_registry.map_imperatively(model.Page, model.Page.table, properties=dict( - user=relation(model.User), - revisions=relation(model.PageRevision, - backref='page', - cascade="all, delete-orphan", - primaryjoin=(model.Page.table.c.id == model.PageRevision.page_id)), - latest_revision=relation(model.PageRevision, - post_update=True, - primaryjoin=(model.Page.table.c.latest_revision_id == model.PageRevision.id), - lazy=False), - tags=relation(model.PageTagAssociation, - order_by=model.PageTagAssociation.id, - back_populates="page"), - annotations=relation(model.PageAnnotationAssociation, - order_by=model.PageAnnotationAssociation.id, - back_populates="page"), - ratings=relation(model.PageRatingAssociation, - order_by=model.PageRatingAssociation.id, - back_populates="page"), - average_rating=column_property( - select(func.avg(model.PageRatingAssociation.rating)).where(model.PageRatingAssociation.page_id == model.Page.table.c.id).scalar_subquery(), - deferred=True - ), - users_shared_with=relation(model.PageUserShareAssociation, back_populates='page') -)) - # Set up proxy so that # Page.users_shared_with # returns a list of users that page is shared with. diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 945da5b1f70..96152339717 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -550,7 +550,7 @@ def test_Page( page_user_share_association, ): cls = model.Page - #assert cls.__tablename__ == 'page_annotation_association' + assert cls.__tablename__ == 'page' assert has_index(cls.__table__, ('slug',)) with dbcleanup(session, cls): title, deleted, importable, slug, published = 'a', True, True, 'b', True @@ -586,7 +586,7 @@ def test_Page( assert stored_obj.importable == importable assert stored_obj.slug == slug assert stored_obj.published == published - # This doesn't test the average; need to use multiple ratings for that. + # This doesn't test the average amount, just the mapping. assert stored_obj.average_rating == page_rating_association.rating # test mapped relationships assert stored_obj.user.id == user.id @@ -598,6 +598,21 @@ def test_Page( assert stored_obj.users_shared_with == [page_user_share_association] +def test_Page_average_rating(model, session, page, user): + cls = model.PageRatingAssociation + with dbcleanup(session, cls): + # Page has been expunged; to access its deferred properties, + # it needs to be added back to the session. + session.add(page) + assert page.average_rating is None # With no ratings, we expect None. + # Create ratings + for rating in (1, 2, 3, 4, 5): + r = cls(user, page) + r.rating = rating + persist(session, r) + assert page.average_rating == 3.0 # Expect average after ratings added. + + def test_PageAnnotationAssociation(model, session, page, user): cls = model.PageAnnotationAssociation assert cls.__tablename__ == 'page_annotation_association' @@ -664,6 +679,8 @@ def test_PageRevision(model, session, page): assert stored_obj.title == title assert stored_obj.content == content assert stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT + # test mapped relationships + assert stored_obj.page.id == page.id def test_PageTagAssociation(model, session, page, tag, user): From f4b473d6a48a6ce523568b9c23892488d896163b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 29 Jun 2021 02:36:08 -0400 Subject: [PATCH 070/225] Work around mypy error --- lib/galaxy/model/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 73e6c678ce0..4263c3047ca 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6630,8 +6630,9 @@ class Page(Base, Dictifiable, RepresentById): users_shared_with = relationship( 'PageUserShareAssociation', back_populates='page') - # `average_rating` added as column_property at the bottom of this module + # `average_rating` added at the bottom of this module # (cannot be added before PageRatingAssociation is defined) + average_rating: column_property dict_element_visible_keys = ['id', 'title', 'latest_revision_id', 'slug', 'published', 'importable', 'deleted', 'username'] From 747b6f35897bd53e08ede658b4c77abfb601b9e2 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 29 Jun 2021 14:04:04 -0400 Subject: [PATCH 071/225] Map FormDefinitionCurrent declaratively --- lib/galaxy/model/__init__.py | 21 ++++++- lib/galaxy/model/mapping.py | 22 ++----- test/unit/model/test_model.py | 105 ++++++++++++++++++++++++++++------ 3 files changed, 113 insertions(+), 35 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4263c3047ca..39f0b01097e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6209,7 +6209,26 @@ class FormDefinition(Dictifiable, RepresentById): return gridfields -class FormDefinitionCurrent(RepresentById): +class FormDefinitionCurrent(Base, RepresentById): + __tablename__ = 'form_definition_current' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + latest_form_id = Column('latest_form_id', Integer, ForeignKey('form_definition.id'), index=True) + deleted = Column('deleted', Boolean, index=True, default=False) + + forms = relationship( + 'FormDefinition', + back_populates='form_definition_current', + cascade='all, delete-orphan', + primaryjoin=('FormDefinitionCurrent.id == FormDefinition.form_definition_current_id')) + + latest_form = relationship( + 'FormDefinition', + post_update=True, + primaryjoin=('FormDefinitionCurrent.latest_form_id == model.FormDefinition.id')) + def __init__(self, form_definition=None): self.latest_form = form_definition diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 5e5947aadc5..fd76a633c74 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1017,14 +1017,6 @@ model.MetadataFile.table = Table( Column("deleted", Boolean, index=True, default=False), Column("purged", Boolean, index=True, default=False)) -model.FormDefinitionCurrent.table = Table( - "form_definition_current", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("latest_form_id", Integer, ForeignKey("form_definition.id"), index=True), - Column("deleted", Boolean, index=True, default=False)) - model.FormDefinition.table = Table( "form_definition", metadata, Column("id", Integer, primary_key=True), @@ -1169,16 +1161,10 @@ mapper_registry.map_imperatively(model.FormValues, model.FormValues.table, prope primaryjoin=(model.FormValues.table.c.form_definition_id == model.FormDefinition.table.c.id)) )) -mapper_registry.map_imperatively(model.FormDefinition, model.FormDefinition.table) - -mapper_registry.map_imperatively(model.FormDefinitionCurrent, model.FormDefinitionCurrent.table, properties=dict( - forms=relation(model.FormDefinition, - backref='form_definition_current', - cascade="all, delete-orphan", - primaryjoin=(model.FormDefinitionCurrent.table.c.id == model.FormDefinition.table.c.form_definition_current_id)), - latest_form=relation(model.FormDefinition, - post_update=True, - primaryjoin=(model.FormDefinitionCurrent.table.c.latest_form_id == model.FormDefinition.table.c.id)) +mapper_registry.map_imperatively(model.FormDefinition, model.FormDefinition.table, properties=dict( + form_definition_current=relation(model.FormDefinitionCurrent, + back_populates='forms', + primaryjoin=(model.FormDefinitionCurrent.id == model.FormDefinition.table.c.form_definition_current_id)), )) simple_mapping(model.HistoryDatasetAssociation, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 96152339717..f8880a0093f 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -12,7 +12,7 @@ from sqlalchemy import ( import galaxy.model.mapping as mapping -def testAPIKeys(model, session, user): +def test_APIKeys(model, session, user): cls = model.APIKeys assert cls.__tablename__ == 'api_keys' with dbcleanup(session, cls): @@ -194,6 +194,67 @@ def test_DynamicTool(model, session, workflow_step): assert workflow_step in stored_obj.workflow_steps +def test_FormDefinition(model, session, form_definition_current): + cls = model.FormDefinition + # assert cls.__tablename__ == 'form_definition' + with dbcleanup(session, cls): + name, desc, fields, type, layout = 'a', 'b', 'c', 'd', 'e' + obj = cls() + obj.name = name + obj.desc = desc + obj.form_definition_current = form_definition_current + obj.fields = fields + obj.type = type + obj.layout = layout + + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.desc == desc + assert stored_obj.form_definition_current_id == form_definition_current.id + assert stored_obj.fields == fields + assert stored_obj.type == type + assert stored_obj.layout == layout + # test mapped relationships + assert stored_obj.form_definition_current.id == form_definition_current.id + + +def test_FormDefinitionCurrent(model, session, form_definition): + cls = model.FormDefinitionCurrent + assert cls.__tablename__ == 'form_definition_current' + with dbcleanup(session, cls): + obj = cls() + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + deleted = True + obj.create_time = create_time + obj.update_time = update_time + obj.latest_form = form_definition + obj.deleted = deleted + obj.forms.append(form_definition) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.latest_form_id == form_definition.id + assert stored_obj.deleted == deleted + # test mapped relationships + assert stored_obj.latest_form.id == form_definition.id + assert stored_obj.forms == [form_definition] + + def test_GroupQuotaAssociation(model, session, group, quota): cls = model.GroupQuotaAssociation assert cls.__tablename__ == 'group_quota_association' @@ -1350,9 +1411,27 @@ def default_quota_association(model, session, quota): @pytest.fixture -def group_role_association(model, session): - gra = model.GroupRoleAssociation(None, None) - yield from dbcleanup_wrapper(session, gra) +def form_definition(model, session, form_definition_current): + fd = model.FormDefinition(name='a', form_definition_current=form_definition_current) + yield from dbcleanup_wrapper(session, fd) + + +@pytest.fixture +def form_definition_current(model, session): + fdc = model.FormDefinitionCurrent() + yield from dbcleanup_wrapper(session, fdc) + + +@pytest.fixture +def galaxy_session(model, session, user): + s = model.GalaxySession() + yield from dbcleanup_wrapper(session, s) + + +@pytest.fixture +def group(model, session): + g = model.Group() + yield from dbcleanup_wrapper(session, g) @pytest.fixture @@ -1361,6 +1440,12 @@ def group_quota_association(model, session): yield from dbcleanup_wrapper(session, gqa) +@pytest.fixture +def group_role_association(model, session): + gra = model.GroupRoleAssociation(None, None) + yield from dbcleanup_wrapper(session, gra) + + @pytest.fixture def history(model, session): h = model.History() @@ -1397,18 +1482,6 @@ def history_tag_association(model, session): yield from dbcleanup_wrapper(session, hta) -@pytest.fixture -def galaxy_session(model, session, user): - s = model.GalaxySession() - yield from dbcleanup_wrapper(session, s) - - -@pytest.fixture -def group(model, session): - g = model.Group() - yield from dbcleanup_wrapper(session, g) - - @pytest.fixture def library_dataset_collection_association(model, session): ldca = model.LibraryDatasetCollectionAssociation() From a6a7413e79f27599ba0839215639376b3d8a5a05 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 29 Jun 2021 18:15:11 -0400 Subject: [PATCH 072/225] Map FormDefinition declaratively --- lib/galaxy/model/__init__.py | 21 ++++++++++++++++++--- lib/galaxy/model/mapping.py | 26 ++++---------------------- test/unit/model/test_model.py | 2 +- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 39f0b01097e..00dc1f5209c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6164,7 +6164,24 @@ class MetadataFile(StorableObject, RepresentById): return as_dict -class FormDefinition(Dictifiable, RepresentById): +class FormDefinition(Base, Dictifiable, RepresentById): + __tablename__ = 'form_definition' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + name = Column('name', TrimmedString(255), nullable=False) + desc = Column('desc', TEXT) + form_definition_current_id = Column('form_definition_current_id', Integer, + ForeignKey('form_definition_current.id', use_alter=True), index=True, nullable=False) + fields = Column('fields', MutableJSONType) + type = Column('type', TrimmedString(255), index=True) + layout = Column('layout', MutableJSONType) + form_definition_current = relationship( + 'FormDefinitionCurrent', + back_populates='forms', + primaryjoin=('FormDefinitionCurrent.id == FormDefinition.form_definition_current_id')) + # The following form_builder classes are supported by the FormDefinition class. supported_field_types = [AddressField, CheckboxField, PasswordField, SelectField, TextArea, TextField, WorkflowField, WorkflowMappingField, HistoryField] @@ -6217,13 +6234,11 @@ class FormDefinitionCurrent(Base, RepresentById): update_time = Column('update_time', DateTime, default=now, onupdate=now) latest_form_id = Column('latest_form_id', Integer, ForeignKey('form_definition.id'), index=True) deleted = Column('deleted', Boolean, index=True, default=False) - forms = relationship( 'FormDefinition', back_populates='form_definition_current', cascade='all, delete-orphan', primaryjoin=('FormDefinitionCurrent.id == FormDefinition.form_definition_current_id')) - latest_form = relationship( 'FormDefinition', post_update=True, diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index fd76a633c74..7557257b83d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1017,18 +1017,6 @@ model.MetadataFile.table = Table( Column("deleted", Boolean, index=True, default=False), Column("purged", Boolean, index=True, default=False)) -model.FormDefinition.table = Table( - "form_definition", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("name", TrimmedString(255), nullable=False), - Column("desc", TEXT), - Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False), - Column("fields", MutableJSONType), - Column("type", TrimmedString(255), index=True), - Column("layout", MutableJSONType)) - model.FormValues.table = Table( "form_values", metadata, Column("id", Integer, primary_key=True), @@ -1158,13 +1146,7 @@ mapper_registry.map_imperatively(model.UserPreference, model.UserPreference.tabl mapper_registry.map_imperatively(model.FormValues, model.FormValues.table, properties=dict( form_definition=relation(model.FormDefinition, - primaryjoin=(model.FormValues.table.c.form_definition_id == model.FormDefinition.table.c.id)) -)) - -mapper_registry.map_imperatively(model.FormDefinition, model.FormDefinition.table, properties=dict( - form_definition_current=relation(model.FormDefinitionCurrent, - back_populates='forms', - primaryjoin=(model.FormDefinitionCurrent.id == model.FormDefinition.table.c.form_definition_current_id)), + primaryjoin=(model.FormValues.table.c.form_definition_id == model.FormDefinition.id)) )) simple_mapping(model.HistoryDatasetAssociation, @@ -1446,7 +1428,7 @@ mapper_registry.map_imperatively(model.LibraryInfoAssociation, model.LibraryInfo ), backref="info_association"), template=relation(model.FormDefinition, - primaryjoin=(model.LibraryInfoAssociation.table.c.form_definition_id == model.FormDefinition.table.c.id)), + primaryjoin=(model.LibraryInfoAssociation.table.c.form_definition_id == model.FormDefinition.id)), info=relation(model.FormValues, primaryjoin=(model.LibraryInfoAssociation.table.c.form_values_id == model.FormValues.table.c.id)) )) @@ -1492,7 +1474,7 @@ mapper_registry.map_imperatively(model.LibraryFolderInfoAssociation, model.Libra ), backref="info_association"), template=relation(model.FormDefinition, - primaryjoin=(model.LibraryFolderInfoAssociation.table.c.form_definition_id == model.FormDefinition.table.c.id)), + primaryjoin=(model.LibraryFolderInfoAssociation.table.c.form_definition_id == model.FormDefinition.id)), info=relation(model.FormValues, primaryjoin=(model.LibraryFolderInfoAssociation.table.c.form_values_id == model.FormValues.table.c.id)) )) @@ -1553,7 +1535,7 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod ), backref="info_association"), template=relation(model.FormDefinition, - primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_definition_id == model.FormDefinition.table.c.id)), + primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_definition_id == model.FormDefinition.id)), info=relation(model.FormValues, primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.table.c.id)) )) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index f8880a0093f..46de1b7f2e9 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -196,7 +196,7 @@ def test_DynamicTool(model, session, workflow_step): def test_FormDefinition(model, session, form_definition_current): cls = model.FormDefinition - # assert cls.__tablename__ == 'form_definition' + assert cls.__tablename__ == 'form_definition' with dbcleanup(session, cls): name, desc, fields, type, layout = 'a', 'b', 'c', 'd', 'e' obj = cls() From 431351e9de121e17748d2251a01f6b83eb193bbc Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 29 Jun 2021 18:55:39 -0400 Subject: [PATCH 073/225] Map FormValues declaratively --- lib/galaxy/model/__init__.py | 13 ++++++++++++- lib/galaxy/model/mapping.py | 21 ++++----------------- test/unit/model/test_model.py | 26 +++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 00dc1f5209c..16c4d7ed2b8 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6248,7 +6248,18 @@ class FormDefinitionCurrent(Base, RepresentById): self.latest_form = form_definition -class FormValues(RepresentById): +class FormValues(Base, RepresentById): + __tablename__ = 'form_values' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + form_definition_id = Column('form_definition_id', Integer, ForeignKey('form_definition.id'), index=True) + content = Column('content', MutableJSONType) + form_definition = relationship( + 'FormDefinition', + primaryjoin=('FormValues.form_definition_id == FormDefinition.id')) + def __init__(self, form_def=None, content=None): self.form_definition = form_def self.content = content diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 7557257b83d..92c52536adb 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1017,14 +1017,6 @@ model.MetadataFile.table = Table( Column("deleted", Boolean, index=True, default=False), Column("purged", Boolean, index=True, default=False)) -model.FormValues.table = Table( - "form_values", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True), - Column("content", MutableJSONType)) - model.Visualization.table = Table( "visualization", metadata, Column("id", Integer, primary_key=True), @@ -1144,11 +1136,6 @@ def simple_mapping(model, **kwds): mapper_registry.map_imperatively(model.UserPreference, model.UserPreference.table, properties={}) -mapper_registry.map_imperatively(model.FormValues, model.FormValues.table, properties=dict( - form_definition=relation(model.FormDefinition, - primaryjoin=(model.FormValues.table.c.form_definition_id == model.FormDefinition.id)) -)) - simple_mapping(model.HistoryDatasetAssociation, dataset=relation(model.Dataset, primaryjoin=(model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id), @@ -1356,7 +1343,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( backref="user", collection_class=attribute_mapped_collection('name')), values=relation(model.FormValues, - primaryjoin=(model.User.table.c.form_values_id == model.FormValues.table.c.id)), + primaryjoin=(model.User.table.c.form_values_id == model.FormValues.id)), api_keys=relation(model.APIKeys, back_populates="user", order_by=desc(model.APIKeys.create_time)), @@ -1430,7 +1417,7 @@ mapper_registry.map_imperatively(model.LibraryInfoAssociation, model.LibraryInfo template=relation(model.FormDefinition, primaryjoin=(model.LibraryInfoAssociation.table.c.form_definition_id == model.FormDefinition.id)), info=relation(model.FormValues, - primaryjoin=(model.LibraryInfoAssociation.table.c.form_values_id == model.FormValues.table.c.id)) + primaryjoin=(model.LibraryInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) mapper_registry.map_imperatively(model.LibraryFolder, model.LibraryFolder.table, properties=dict( @@ -1476,7 +1463,7 @@ mapper_registry.map_imperatively(model.LibraryFolderInfoAssociation, model.Libra template=relation(model.FormDefinition, primaryjoin=(model.LibraryFolderInfoAssociation.table.c.form_definition_id == model.FormDefinition.id)), info=relation(model.FormValues, - primaryjoin=(model.LibraryFolderInfoAssociation.table.c.form_values_id == model.FormValues.table.c.id)) + primaryjoin=(model.LibraryFolderInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) mapper_registry.map_imperatively(model.LibraryDataset, model.LibraryDataset.table, properties=dict( @@ -1537,7 +1524,7 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod template=relation(model.FormDefinition, primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_definition_id == model.FormDefinition.id)), info=relation(model.FormValues, - primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.table.c.id)) + primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) mapper_registry.map_imperatively(model.JobToInputDatasetAssociation, model.JobToInputDatasetAssociation.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 46de1b7f2e9..2e0cfec82f9 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -206,7 +206,6 @@ def test_FormDefinition(model, session, form_definition_current): obj.fields = fields obj.type = type obj.layout = layout - create_time = datetime.now() update_time = create_time + timedelta(hours=1) obj.create_time = create_time @@ -255,6 +254,31 @@ def test_FormDefinitionCurrent(model, session, form_definition): assert stored_obj.forms == [form_definition] +def test_FormValues(model, session, form_definition): + cls = model.FormValues + assert cls.__tablename__ == 'form_values' + with dbcleanup(session, cls): + content = 'a' + obj = cls() + obj.form_definition = form_definition + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.content = content + obj.create_time = create_time + obj.update_time = update_time + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.content == content + assert stored_obj.form_definition_id == form_definition.id + # test mapped relationships + assert stored_obj.form_definition.id == form_definition.id + + def test_GroupQuotaAssociation(model, session, group, quota): cls = model.GroupQuotaAssociation assert cls.__tablename__ == 'group_quota_association' From 5e1d04a38b7eccd6d15a05a470d88052f8d0d669 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 30 Jun 2021 15:52:04 -0400 Subject: [PATCH 074/225] Fix HistoryAudit bug, add test, modify helpers (see note) 1. Bug: HistoryAudit class was unmapped. Therefore, any persistence-related code like the following would raise an error: session.add(HistoryAudit(history, update_time)) (This is why we should never add a model class without a basic mapping test.) Test exposing the bug added. 2. I've temporarily mapped the class using classical mapping - this will be replaced w/declarative in a subsequent commit. 3. Modify helper functions to accommodate models that do not use `id` as their pkey. --- lib/galaxy/model/mapping.py | 4 ++++ test/unit/model/test_model.py | 29 +++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 92c52536adb..951416c5e9b 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -105,6 +105,10 @@ model.HistoryAudit.table = Table( PrimaryKeyConstraint(sqlite_on_conflict='IGNORE') ) +# Add this temporarily as a bug fix (class was unmapped); remove upon remapping declaratively +mapper_registry.map_imperatively(model.HistoryAudit, model.HistoryAudit.table, properties=dict( + history=relation(model.History))) + model.HistoryUserShareAssociation.table = Table( "history_user_share_association", metadata, Column("id", Integer, primary_key=True), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 2e0cfec82f9..08de63ae721 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -325,6 +325,24 @@ def test_GroupRoleAssociation(model, session, group, role): assert stored_obj.role.id == role.id +def test_HistoryAudit(model, session, history): + cls = model.HistoryAudit + # TODO: assert cls.__tablename__ == 'history_audit' + # TODO: assert pkey constraint + with dbcleanup(session, cls): + update_time = datetime.now() + obj = cls(history, update_time) + persist(session, obj, return_id=False) + + stmt = select(cls).where(cls.history_id == history.id, cls.update_time == update_time) + stored_obj = get_stored_obj(session, cls, stmt=stmt) + # test mapped columns + assert stored_obj.history_id == history.id + assert stored_obj.update_time == update_time + # test mapped relationships + assert stored_obj.history.id == history.id + + def test_HistoryAnnotationAssociation(model, session, history, user): cls = model.HistoryAnnotationAssociation assert cls.__tablename__ == 'history_annotation_association' @@ -1672,10 +1690,10 @@ def dbcleanup_wrapper(session, obj): yield obj -def persist(session, obj): +def persist(session, obj, return_id=True): session.add(obj) session.flush() - obj_id = obj.id + obj_id = obj.id if return_id else None # save this before obj is expunged # Remove from session, so that on a subsequent load we get a *new* obj from the db session.expunge(obj) return obj_id @@ -1696,8 +1714,11 @@ def has_index(table, fields): return True -def get_stored_obj(session, cls, obj_id): - stmt = select(cls).filter(cls.id == obj_id) +def get_stored_obj(session, cls, obj_id=None, stmt=None): + # Either obj_id or stmt must be provided, but not both + assert bool(obj_id) ^ (stmt is not None) + if stmt is None: + stmt = select(cls).where(cls.id == obj_id) return session.execute(stmt).scalar_one() From 676d2578f527b331e7f371c1c0eeebc3c40dac56 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 30 Jun 2021 16:57:33 -0400 Subject: [PATCH 075/225] Map HistoryAudit declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 14 +------------- test/unit/model/test_model.py | 3 +-- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 16c4d7ed2b8..c0e3bf0a968 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -46,6 +46,7 @@ from sqlalchemy import ( join, not_, or_, + PrimaryKeyConstraint, select, String, TEXT, @@ -1933,7 +1934,16 @@ def is_hda(d): return isinstance(d, HistoryDatasetAssociation) -class HistoryAudit(RepresentById): +class HistoryAudit(Base, RepresentById): + __tablename__ = 'history_audit' + __table_args__ = ( + PrimaryKeyConstraint(sqlite_on_conflict='IGNORE'), + ) + + history_id = Column('history_id', Integer, ForeignKey('history.id'), primary_key=True, nullable=False) + update_time = Column('update_time', DateTime, default=now, primary_key=True, nullable=False) + history = relationship('History') + def __init__(self, history, update_time): self.history = history self.update_time = update_time diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 951416c5e9b..ec6d7651bda 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -21,7 +21,6 @@ from sqlalchemy import ( Integer, not_, Numeric, - PrimaryKeyConstraint, select, String, Table, TEXT, @@ -98,17 +97,6 @@ model.History.table = Table( Index('ix_history_slug', 'slug', mysql_length=200), ) -model.HistoryAudit.table = Table( - "history_audit", metadata, - Column("history_id", Integer, ForeignKey("history.id"), primary_key=True, nullable=False), - Column("update_time", DateTime, default=now, primary_key=True, nullable=False), - PrimaryKeyConstraint(sqlite_on_conflict='IGNORE') -) - -# Add this temporarily as a bug fix (class was unmapped); remove upon remapping declaratively -mapper_registry.map_imperatively(model.HistoryAudit, model.HistoryAudit.table, properties=dict( - history=relation(model.History))) - model.HistoryUserShareAssociation.table = Table( "history_user_share_association", metadata, Column("id", Integer, primary_key=True), @@ -1296,7 +1284,7 @@ mapper_registry.map_imperatively(model.History, model.History.table, properties= deferred=True ), update_time=column_property( - select(func.max(model.HistoryAudit.table.c.update_time)).where(model.HistoryAudit.table.c.history_id == model.History.table.c.id).scalar_subquery(), + select(func.max(model.HistoryAudit.update_time)).where(model.HistoryAudit.history_id == model.History.table.c.id).scalar_subquery(), ), )) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 08de63ae721..31103bebfcd 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -327,8 +327,7 @@ def test_GroupRoleAssociation(model, session, group, role): def test_HistoryAudit(model, session, history): cls = model.HistoryAudit - # TODO: assert cls.__tablename__ == 'history_audit' - # TODO: assert pkey constraint + assert cls.__tablename__ == 'history_audit' with dbcleanup(session, cls): update_time = datetime.now() obj = cls(history, update_time) From 1d7025ceedc6f989936197256041acf14536ca72 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 30 Jun 2021 17:16:48 -0400 Subject: [PATCH 076/225] Map PasswordResetToken declaratively --- lib/galaxy/model/__init__.py | 9 ++++++++- lib/galaxy/model/mapping.py | 11 +---------- test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index c0e3bf0a968..6a24ecb9920 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -684,7 +684,14 @@ class User(Dictifiable, RepresentById): session.flush() -class PasswordResetToken(_HasTable): +class PasswordResetToken(Base, _HasTable): + __tablename__ = 'password_reset_token' + + token = Column('token', String(32), primary_key=True, unique=True, index=True) + expiration_time = Column('expiration_time', DateTime) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user = relationship('User', back_populates='reset_tokens') + def __init__(self, user, token=None): if token: self.token = token diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index ec6d7651bda..d64608940a2 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -73,12 +73,6 @@ model.User.table = Table( Column("active", Boolean, index=True, default=True, nullable=False), Column("activation_token", TrimmedString(64), nullable=True, index=True)) -model.PasswordResetToken.table = Table( - "password_reset_token", metadata, - Column("token", String(32), primary_key=True, unique=True, index=True), - Column("expiration_time", DateTime), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True)) - model.History.table = Table( "history", metadata, Column("id", Integer, primary_key=True), @@ -1340,12 +1334,9 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( back_populates="user", order_by=desc(model.APIKeys.create_time)), pages=relation(model.Page, back_populates='user'), + reset_tokens=relation(model.PasswordResetToken, back_populates='user'), )) -mapper_registry.map_imperatively(model.PasswordResetToken, model.PasswordResetToken.table, - properties=dict(user=relation(model.User, backref="reset_tokens"))) - - # Set up proxy so that this syntax is possible: # .preferences[pref_name] = pref_value model.User.preferences = association_proxy('_preferences', 'value', creator=model.UserPreference) # type: ignore diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 31103bebfcd..c28ec343653 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -829,6 +829,25 @@ def test_PageUserShareAssociation(model, session, page, user): assert stored_obj.user.id == user.id +def test_PasswordResetToken(model, session, user): + cls = model.PasswordResetToken + assert cls.__tablename__ == 'password_reset_token' + with dbcleanup(session, cls): + token = get_random_string() + expiration_time = datetime.now() + obj = cls(user, token) + obj.expiration_time = expiration_time + persist(session, obj, return_id=False) + + stmt = select(cls).where(cls.token == token) + stored_obj = get_stored_obj(session, cls, stmt=stmt) + # test mapped columns + assert stored_obj.token == token + assert stored_obj.expiration_time == expiration_time + # test mapped relationships + assert stored_obj.user.id == user.id + + def test_PSAAssociation(model, session): cls = model.PSAAssociation assert cls.__tablename__ == 'psa_association' From 0ecbf34b567ec72e1ed4579f3865ba31ae30d5af Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 30 Jun 2021 17:31:56 -0400 Subject: [PATCH 077/225] Map DatasetHash declaratively --- lib/galaxy/model/__init__.py | 11 +++++++++-- lib/galaxy/model/mapping.py | 14 +------------- test/unit/model/test_model.py | 23 +++++++++++++++++++++++ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6a24ecb9920..acab2c08f92 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2914,8 +2914,15 @@ class DatasetSourceHash(RepresentById): """ """ -class DatasetHash(RepresentById): - """ """ +class DatasetHash(Base, RepresentById): + __tablename__ = 'dataset_hash' + + id = Column('id', Integer, primary_key=True) + dataset_id = Column('dataset_id', Integer, ForeignKey('dataset.id'), index=True) + hash_function = Column('hash_function', TEXT) + hash_value = Column('hash_value', TEXT) + extra_files_path = Column('extra_files_path', TEXT) + dataset = relationship('Dataset', back_populates='hashes') def serialize(self, id_encoder, serialization_options): # serialize Dataset objects only for jobs that can actually modify these models. diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index d64608940a2..f1c9178fda0 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -170,15 +170,6 @@ model.DatasetSource.table = Table( Column("transform", MutableJSONType) ) -model.DatasetHash.table = Table( - "dataset_hash", metadata, - Column("id", Integer, primary_key=True), - Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), - Column("hash_function", TEXT), - Column("hash_value", TEXT), - Column("extra_files_path", TEXT), -) - model.DatasetSourceHash.table = Table( "dataset_source_hash", metadata, Column("id", Integer, primary_key=True), @@ -1177,12 +1168,9 @@ simple_mapping(model.Dataset, (model.Dataset.table.c.id == model.LibraryDatasetDatasetAssociation.table.c.dataset_id) & (model.LibraryDatasetDatasetAssociation.table.c.deleted == false())), viewonly=True), + hashes=relation(model.DatasetHash, back_populates='dataset'), ) -mapper_registry.map_imperatively(model.DatasetHash, model.DatasetHash.table, properties=dict( - dataset=relation(model.Dataset, backref='hashes') -)) - mapper_registry.map_imperatively(model.DatasetSource, model.DatasetSource.table, properties=dict( dataset=relation(model.Dataset, backref='sources') )) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index c28ec343653..7ae1266b695 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -87,6 +87,29 @@ def test_CustosAuthnzToken(model, session, user): assert stored_obj.user.id == user.id +def test_DatasetHash(model, session, dataset): + cls = model.DatasetHash + assert cls.__tablename__ == 'dataset_hash' + with dbcleanup(session, cls): + hash_function, hash_value, extra_files_path = 'a', 'b', 'c' + obj = cls() + obj.dataset = dataset + obj.hash_function = hash_function + obj.hash_value = hash_value + obj.extra_files_path = extra_files_path + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.dataset_id == dataset.id + assert stored_obj.hash_function == hash_function + assert stored_obj.hash_value == hash_value + assert stored_obj.extra_files_path == extra_files_path + # test mapped relationships + assert stored_obj.dataset.id == dataset.id + + def test_DatasetPermissions(model, session, dataset, role): cls = model.DatasetPermissions assert cls.__tablename__ == 'dataset_permissions' From 2377c33482e2171a873c216a193dd353a23a2a4a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 30 Jun 2021 17:38:42 -0400 Subject: [PATCH 078/225] Map DatasetSources declaratively --- lib/galaxy/model/__init__.py | 11 +++++++++-- lib/galaxy/model/mapping.py | 14 +------------- test/unit/model/test_model.py | 23 +++++++++++++++++++++++ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index acab2c08f92..73fc4dfd1d1 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2906,8 +2906,15 @@ class Dataset(StorableObject, RepresentById, _HasTable): return rval -class DatasetSource(RepresentById): - """ """ +class DatasetSource(Base, RepresentById): + __tablename__ = 'dataset_source' + + id = Column('id', Integer, primary_key=True) + dataset_id = Column('dataset_id', Integer, ForeignKey('dataset.id'), index=True) + source_uri = Column("source_uri", TEXT) + extra_files_path = Column("extra_files_path", TEXT) + transform = Column("transform", MutableJSONType) + dataset = relationship('Dataset', back_populates='sources') class DatasetSourceHash(RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f1c9178fda0..03bc1778d16 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -161,15 +161,6 @@ model.Dataset.table = Table( Column('total_size', Numeric(15, 0)), Column('uuid', UUIDType())) -model.DatasetSource.table = Table( - "dataset_source", metadata, - Column("id", Integer, primary_key=True), - Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), - Column("source_uri", TEXT), - Column("extra_files_path", TEXT), - Column("transform", MutableJSONType) -) - model.DatasetSourceHash.table = Table( "dataset_source_hash", metadata, Column("id", Integer, primary_key=True), @@ -1169,12 +1160,9 @@ simple_mapping(model.Dataset, & (model.LibraryDatasetDatasetAssociation.table.c.deleted == false())), viewonly=True), hashes=relation(model.DatasetHash, back_populates='dataset'), + sources=relation(model.DatasetSource, back_populates='dataset'), ) -mapper_registry.map_imperatively(model.DatasetSource, model.DatasetSource.table, properties=dict( - dataset=relation(model.Dataset, backref='sources') -)) - mapper_registry.map_imperatively(model.DatasetSourceHash, model.DatasetSourceHash.table, properties=dict( source=relation(model.DatasetSource, backref='hashes') )) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 7ae1266b695..1e9134d6d8c 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -110,6 +110,29 @@ def test_DatasetHash(model, session, dataset): assert stored_obj.dataset.id == dataset.id +def test_DatasetSource(model, session, dataset): + cls = model.DatasetSource + assert cls.__tablename__ == 'dataset_source' + with dbcleanup(session, cls): + source_uri, extra_files_path, transform = 'a', 'b', 'c' + obj = cls() + obj.dataset = dataset + obj.source_uri = source_uri + obj.extra_files_path = extra_files_path + obj.transform = transform + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.dataset_id == dataset.id + assert stored_obj.source_uri == source_uri + assert stored_obj.extra_files_path == extra_files_path + assert stored_obj.transform == transform + # test mapped relationships + assert stored_obj.dataset.id == dataset.id + + def test_DatasetPermissions(model, session, dataset, role): cls = model.DatasetPermissions assert cls.__tablename__ == 'dataset_permissions' From 9860a94f49026118c8c0c5862a27d347f39018c7 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 30 Jun 2021 17:50:22 -0400 Subject: [PATCH 079/225] Map DatasetSourceHash declaratively --- lib/galaxy/model/__init__.py | 11 +++++++++-- lib/galaxy/model/mapping.py | 12 ------------ test/unit/model/test_model.py | 37 ++++++++++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 73fc4dfd1d1..26f102669ce 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2915,10 +2915,17 @@ class DatasetSource(Base, RepresentById): extra_files_path = Column("extra_files_path", TEXT) transform = Column("transform", MutableJSONType) dataset = relationship('Dataset', back_populates='sources') + hashes = relationship('DatasetSourceHash', back_populates='source') -class DatasetSourceHash(RepresentById): - """ """ +class DatasetSourceHash(Base, RepresentById): + __tablename__ = 'dataset_source_hash' + + id = Column('id', Integer, primary_key=True) + dataset_source_id = Column('dataset_source_id', Integer, ForeignKey('dataset_source.id'), index=True) + hash_function = Column('hash_function', TEXT) + hash_value = Column('hash_value', TEXT) + source = relationship('DatasetSource', back_populates='hashes') class DatasetHash(Base, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 03bc1778d16..5dd7471a7f0 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -161,14 +161,6 @@ model.Dataset.table = Table( Column('total_size', Numeric(15, 0)), Column('uuid', UUIDType())) -model.DatasetSourceHash.table = Table( - "dataset_source_hash", metadata, - Column("id", Integer, primary_key=True), - Column("dataset_source_id", Integer, ForeignKey("dataset_source.id"), index=True), - Column("hash_function", TEXT), - Column("hash_value", TEXT) -) - # hda read access permission given by a user to a specific site (gen. for external display applications) model.HistoryDatasetAssociationDisplayAtAuthorization.table = Table( "history_dataset_association_display_at_authorization", metadata, @@ -1163,10 +1155,6 @@ simple_mapping(model.Dataset, sources=relation(model.DatasetSource, back_populates='dataset'), ) -mapper_registry.map_imperatively(model.DatasetSourceHash, model.DatasetSourceHash.table, properties=dict( - source=relation(model.DatasetSource, backref='hashes') -)) - mapper_registry.map_imperatively(model.HistoryDatasetAssociationHistory, model.HistoryDatasetAssociationHistory.table) mapper_registry.map_imperatively(model.HistoryDatasetAssociationDisplayAtAuthorization, model.HistoryDatasetAssociationDisplayAtAuthorization.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 1e9134d6d8c..3a2dbd36887 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -110,7 +110,7 @@ def test_DatasetHash(model, session, dataset): assert stored_obj.dataset.id == dataset.id -def test_DatasetSource(model, session, dataset): +def test_DatasetSource(model, session, dataset, dataset_source_hash): cls = model.DatasetSource assert cls.__tablename__ == 'dataset_source' with dbcleanup(session, cls): @@ -120,6 +120,7 @@ def test_DatasetSource(model, session, dataset): obj.source_uri = source_uri obj.extra_files_path = extra_files_path obj.transform = transform + obj.hashes.append(dataset_source_hash) obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) @@ -131,6 +132,28 @@ def test_DatasetSource(model, session, dataset): assert stored_obj.transform == transform # test mapped relationships assert stored_obj.dataset.id == dataset.id + assert stored_obj.hashes == [dataset_source_hash] + + +def test_DatasetSourceHash(model, session, dataset_source): + cls = model.DatasetSourceHash + assert cls.__tablename__ == 'dataset_source_hash' + with dbcleanup(session, cls): + hash_function, hash_value = 'a', 'b' + obj = cls() + obj.source = dataset_source + obj.hash_function = hash_function + obj.hash_value = hash_value + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.dataset_source_id == dataset_source.id + assert stored_obj.hash_function == hash_function + assert stored_obj.hash_value == hash_value + # test mapped relationships + assert stored_obj.source.id == dataset_source.id def test_DatasetPermissions(model, session, dataset, role): @@ -1509,6 +1532,18 @@ def dataset_permission(model, session, dataset): yield from dbcleanup_wrapper(session, d) +@pytest.fixture +def dataset_source(model, session): + d = model.DatasetSource() + yield from dbcleanup_wrapper(session, d) + + +@pytest.fixture +def dataset_source_hash(model, session): + d = model.DatasetSourceHash() + yield from dbcleanup_wrapper(session, d) + + @pytest.fixture def default_quota_association(model, session, quota): type_ = model.DefaultQuotaAssociation.types.REGISTERED From ecb0b05c6a14cd66b07a815e0d2d84763c09b5e7 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 30 Jun 2021 23:13:10 -0400 Subject: [PATCH 080/225] Map LibraryPermissions declaratively --- lib/galaxy/model/__init__.py | 14 +++++++++++- lib/galaxy/model/mapping.py | 17 ++------------- test/unit/model/test_model.py | 41 ++++++++++++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 26f102669ce..96cbbb7508c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2412,6 +2412,7 @@ class Role(Base, Dictifiable, RepresentById): deleted = Column('deleted', Boolean, index=True, default=False) dataset_actions = relationship('DatasetPermissions', back_populates='role') groups = relationship('GroupRoleAssociation', back_populates='role') + library_actions = relationship('LibraryPermissions', back_populates='role') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'name', 'description', 'type'] @@ -2558,7 +2559,18 @@ class DatasetPermissions(Base, RepresentById): self.role_id = role_id -class LibraryPermissions(RepresentById): +class LibraryPermissions(Base, RepresentById): + __tablename__ = 'library_permissions' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + action = Column('action', TEXT) + library_id = Column('library_id', Integer, ForeignKey('library.id'), nullable=True, index=True) + role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + library = relationship('Library', back_populates='actions') + role = relationship('Role', back_populates='library_actions') + def __init__(self, action, library_item, role): self.action = action if isinstance(library_item, Library): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 5dd7471a7f0..bd01e16ed32 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -199,15 +199,6 @@ model.UserRoleAssociation.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now)) -model.LibraryPermissions.table = Table( - "library_permissions", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("action", TEXT), - Column("library_id", Integer, ForeignKey("library.id"), nullable=True, index=True), - Column("role_id", Integer, ForeignKey("role.id"), index=True)) - model.LibraryFolderPermissions.table = Table( "library_folder_permissions", metadata, Column("id", Integer, primary_key=True), @@ -1324,11 +1315,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.LibraryPermissions, model.LibraryPermissions.table, properties=dict( - library=relation(model.Library, backref="actions"), - role=relation(model.Role, backref="library_actions") -)) - mapper_registry.map_imperatively(model.LibraryFolderPermissions, model.LibraryFolderPermissions.table, properties=dict( folder=relation(model.LibraryFolder, backref="actions"), role=relation(model.Role, backref="library_folder_actions") @@ -1345,7 +1331,8 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociationPermissio )) mapper_registry.map_imperatively(model.Library, model.Library.table, properties=dict( - root_folder=relation(model.LibraryFolder, backref=backref("library_root")) + root_folder=relation(model.LibraryFolder, backref=backref("library_root")), + actions=relation(model.LibraryPermissions, back_populates='library'), )) mapper_registry.map_imperatively(model.ExtendedMetadata, model.ExtendedMetadata.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 3a2dbd36887..6c399d4a037 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -710,6 +710,31 @@ def test_LibraryDatasetDatasetAssociationTagAssociation( assert stored_obj.user.id == user.id +def test_LibraryPermissions(model, session, library, role): + cls = model.LibraryPermissions + assert cls.__tablename__ == 'library_permissions' + with dbcleanup(session, cls): + action = 'a' + obj = cls(action, library, role) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.action == action + assert stored_obj.library_id == library.id + assert stored_obj.role_id == role.id + # test mapped relationships + assert stored_obj.library.id == library.id + assert stored_obj.role.id == role.id + + def test_Page( model, session, @@ -1028,7 +1053,7 @@ def test_Quota( assert stored_obj.users == [user_quota_association] -def test_Role(model, session, dataset_permission, group_role_association): +def test_Role(model, session, dataset_permission, group_role_association, library_permission): cls = model.Role assert cls.__tablename__ == 'role' with dbcleanup(session, cls): @@ -1039,6 +1064,7 @@ def test_Role(model, session, dataset_permission, group_role_association): obj.create_time = create_time obj.update_time = update_time obj.dataset_actions.append(dataset_permission) + obj.library_actions.append(library_permission) obj.groups.append(group_role_association) obj_id = persist(session, obj) @@ -1054,6 +1080,7 @@ def test_Role(model, session, dataset_permission, group_role_association): # test mapped relationships assert stored_obj.dataset_actions == [dataset_permission] assert stored_obj.groups == [group_role_association] + assert stored_obj.library_actions == [library_permission] def test_StoredWorkflowAnnotationAssociation(model, session, stored_workflow, user): @@ -1623,6 +1650,12 @@ def history_tag_association(model, session): yield from dbcleanup_wrapper(session, hta) +@pytest.fixture +def library(model, session): + lb = model.Library() + yield from dbcleanup_wrapper(session, lb) + + @pytest.fixture def library_dataset_collection_association(model, session): ldca = model.LibraryDatasetCollectionAssociation() @@ -1647,6 +1680,12 @@ def library_dataset_dataset_association_tag_association(model, session): yield from dbcleanup_wrapper(session, lddata) +@pytest.fixture +def library_permission(model, session, library, role): + lp = model.LibraryPermissions('a', library, role) + yield from dbcleanup_wrapper(session, lp) + + @pytest.fixture def page(model, session, user): p = model.Page() From bae0978d5c6e7cea4bf0262afa48fad7574fadae Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 30 Jun 2021 23:46:40 -0400 Subject: [PATCH 081/225] Map Library declaratively --- lib/galaxy/model/__init__.py | 16 ++++++++++++++- lib/galaxy/model/mapping.py | 22 +++------------------ test/unit/model/test_model.py | 37 +++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 96cbbb7508c..504509dab1b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3878,7 +3878,21 @@ class HistoryDatasetAssociationSubset(RepresentById): self.location = location -class Library(Dictifiable, HasName, RepresentById): +class Library(Base, Dictifiable, HasName, RepresentById): + __tablename__ = 'library' + + id = Column("id", Integer, primary_key=True) + root_folder_id = Column("root_folder_id", Integer, ForeignKey("library_folder.id"), index=True) + create_time = Column("create_time", DateTime, default=now) + update_time = Column("update_time", DateTime, default=now, onupdate=now) + name = Column("name", String(255), index=True) + deleted = Column("deleted", Boolean, index=True, default=False) + purged = Column("purged", Boolean, index=True, default=False) + description = Column("description", TEXT) + synopsis = Column("synopsis", TEXT) + root_folder = relationship('LibraryFolder', back_populates='library_root') + actions = relationship('LibraryPermissions', back_populates='library') + permitted_actions = get_permitted_actions(filter='LIBRARY') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'deleted', 'name', 'description', 'synopsis', 'root_folder_id', 'create_time'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index bd01e16ed32..22384efd16d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -300,18 +300,6 @@ model.ExtendedMetadataIndex.table = Table( Column("path", String(255)), Column("value", TEXT)) -model.Library.table = Table( - "library", metadata, - Column("id", Integer, primary_key=True), - Column("root_folder_id", Integer, ForeignKey("library_folder.id"), index=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("name", String(255), index=True), - Column("deleted", Boolean, index=True, default=False), - Column("purged", Boolean, index=True, default=False), - Column("description", TEXT), - Column("synopsis", TEXT)) - model.LibraryFolder.table = Table( "library_folder", metadata, Column("id", Integer, primary_key=True), @@ -1330,11 +1318,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociationPermissio role=relation(model.Role, backref="library_dataset_dataset_actions") )) -mapper_registry.map_imperatively(model.Library, model.Library.table, properties=dict( - root_folder=relation(model.LibraryFolder, backref=backref("library_root")), - actions=relation(model.LibraryPermissions, back_populates='library'), -)) - mapper_registry.map_imperatively(model.ExtendedMetadata, model.ExtendedMetadata.table, properties=dict( children=relation(model.ExtendedMetadataIndex, backref='extended_metadata') )) @@ -1344,7 +1327,7 @@ mapper_registry.map_imperatively(model.ExtendedMetadataIndex, model.ExtendedMeta mapper_registry.map_imperatively(model.LibraryInfoAssociation, model.LibraryInfoAssociation.table, properties=dict( library=relation(model.Library, primaryjoin=( - (model.LibraryInfoAssociation.table.c.library_id == model.Library.table.c.id) + (model.LibraryInfoAssociation.table.c.library_id == model.Library.id) & (not_(model.LibraryInfoAssociation.table.c.deleted)) ), backref="info_association"), @@ -1384,7 +1367,8 @@ mapper_registry.map_imperatively(model.LibraryFolder, model.LibraryFolder.table, ), order_by=asc(model.LibraryDataset.table.c._name), lazy=True, - viewonly=True) + viewonly=True), + library_root=relation(model.Library, back_populates='root_folder'), )) mapper_registry.map_imperatively(model.LibraryFolderInfoAssociation, model.LibraryFolderInfoAssociation.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 6c399d4a037..2f389ff6412 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -616,6 +616,37 @@ def test_HistoryTagAssociation(model, session, history, tag, user): assert stored_obj.user.id == user.id +def test_Library(model, session, library_folder, library_permission): + cls = model.Library + assert cls.__tablename__ == 'library' + with dbcleanup(session, cls): + name, deleted, purged, description, synopsis = 'a', True, True, 'b', 'c' + obj = cls(name, description, synopsis, library_folder) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + obj.deleted = deleted + obj.purged = purged + obj.actions.append(library_permission) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.root_folder_id == library_folder.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + assert stored_obj.description == description + assert stored_obj.synopsis == synopsis + # test mapped relationships + assert stored_obj.root_folder.id == library_folder.id + assert stored_obj.actions == [library_permission] + + def test_LibraryDatasetCollectionAnnotationAssociation( model, session, library_dataset_collection_association, user): cls = model.LibraryDatasetCollectionAnnotationAssociation @@ -1656,6 +1687,12 @@ def library(model, session): yield from dbcleanup_wrapper(session, lb) +@pytest.fixture +def library_folder(model, session): + lf = model.LibraryFolder() + yield from dbcleanup_wrapper(session, lf) + + @pytest.fixture def library_dataset_collection_association(model, session): ldca = model.LibraryDatasetCollectionAssociation() From 358acf8a8fb812b7e593b11cb004f305552d3717 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 01:02:44 -0400 Subject: [PATCH 082/225] Map LibraryFolderPermissions declaratively --- lib/galaxy/model/__init__.py | 14 +++++++++++- lib/galaxy/model/mapping.py | 15 +------------ test/unit/model/test_model.py | 42 ++++++++++++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 504509dab1b..65607f23404 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2413,6 +2413,7 @@ class Role(Base, Dictifiable, RepresentById): dataset_actions = relationship('DatasetPermissions', back_populates='role') groups = relationship('GroupRoleAssociation', back_populates='role') library_actions = relationship('LibraryPermissions', back_populates='role') + library_folder_actions = relationship('LibraryFolderPermissions', back_populates='role') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'name', 'description', 'type'] @@ -2580,7 +2581,18 @@ class LibraryPermissions(Base, RepresentById): self.role = role -class LibraryFolderPermissions(RepresentById): +class LibraryFolderPermissions(Base, RepresentById): + __tablename__ = 'library_folder_permissions' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + action = Column('action', TEXT) + library_folder_id = Column('library_folder_id', Integer, ForeignKey('library_folder.id'), nullable=True, index=True) + role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + folder = relationship('LibraryFolder', back_populates='actions') + role = relationship('Role', back_populates='library_folder_actions') + def __init__(self, action, library_item, role): self.action = action if isinstance(library_item, LibraryFolder): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 22384efd16d..9774b44e1e7 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -199,15 +199,6 @@ model.UserRoleAssociation.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now)) -model.LibraryFolderPermissions.table = Table( - "library_folder_permissions", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("action", TEXT), - Column("library_folder_id", Integer, ForeignKey("library_folder.id"), nullable=True, index=True), - Column("role_id", Integer, ForeignKey("role.id"), index=True)) - model.LibraryDatasetPermissions.table = Table( "library_dataset_permissions", metadata, Column("id", Integer, primary_key=True), @@ -1303,11 +1294,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.LibraryFolderPermissions, model.LibraryFolderPermissions.table, properties=dict( - folder=relation(model.LibraryFolder, backref="actions"), - role=relation(model.Role, backref="library_folder_actions") -)) - mapper_registry.map_imperatively(model.LibraryDatasetPermissions, model.LibraryDatasetPermissions.table, properties=dict( library_dataset=relation(model.LibraryDataset, backref="actions"), role=relation(model.Role, backref="library_dataset_actions") @@ -1369,6 +1355,7 @@ mapper_registry.map_imperatively(model.LibraryFolder, model.LibraryFolder.table, lazy=True, viewonly=True), library_root=relation(model.Library, back_populates='root_folder'), + actions=relation(model.LibraryFolderPermissions, back_populates='folder'), )) mapper_registry.map_imperatively(model.LibraryFolderInfoAssociation, model.LibraryFolderInfoAssociation.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 2f389ff6412..c3725e824d8 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -741,6 +741,31 @@ def test_LibraryDatasetDatasetAssociationTagAssociation( assert stored_obj.user.id == user.id +def test_LibraryFolderPermissions(model, session, library_folder, role): + cls = model.LibraryFolderPermissions + assert cls.__tablename__ == 'library_folder_permissions' + with dbcleanup(session, cls): + action = 'a' + obj = cls(action, library_folder, role) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.action == action + assert stored_obj.library_folder_id == library_folder.id + assert stored_obj.role_id == role.id + # test mapped relationships + assert stored_obj.folder.id == library_folder.id + assert stored_obj.role.id == role.id + + def test_LibraryPermissions(model, session, library, role): cls = model.LibraryPermissions assert cls.__tablename__ == 'library_permissions' @@ -1084,7 +1109,14 @@ def test_Quota( assert stored_obj.users == [user_quota_association] -def test_Role(model, session, dataset_permission, group_role_association, library_permission): +def test_Role( + model, + session, + dataset_permission, + group_role_association, + library_permission, + library_folder_permission, +): cls = model.Role assert cls.__tablename__ == 'role' with dbcleanup(session, cls): @@ -1096,6 +1128,7 @@ def test_Role(model, session, dataset_permission, group_role_association, librar obj.update_time = update_time obj.dataset_actions.append(dataset_permission) obj.library_actions.append(library_permission) + obj.library_folder_actions.append(library_folder_permission) obj.groups.append(group_role_association) obj_id = persist(session, obj) @@ -1112,6 +1145,7 @@ def test_Role(model, session, dataset_permission, group_role_association, librar assert stored_obj.dataset_actions == [dataset_permission] assert stored_obj.groups == [group_role_association] assert stored_obj.library_actions == [library_permission] + assert stored_obj.library_folder_actions == [library_folder_permission] def test_StoredWorkflowAnnotationAssociation(model, session, stored_workflow, user): @@ -1717,6 +1751,12 @@ def library_dataset_dataset_association_tag_association(model, session): yield from dbcleanup_wrapper(session, lddata) +@pytest.fixture +def library_folder_permission(model, session, library_folder, role): + lfp = model.LibraryFolderPermissions('a', library_folder, role) + yield from dbcleanup_wrapper(session, lfp) + + @pytest.fixture def library_permission(model, session, library, role): lp = model.LibraryPermissions('a', library, role) From 25e7a81cdecaf0d02e4f25cd1dd432d5c0af1096 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 01:12:51 -0400 Subject: [PATCH 083/225] Map LibraryDatasetPermissions declaratively --- lib/galaxy/model/__init__.py | 14 +++++++++++- lib/galaxy/model/mapping.py | 17 ++------------- test/unit/model/test_model.py | 40 +++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 65607f23404..ff5eb228e5e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2414,6 +2414,7 @@ class Role(Base, Dictifiable, RepresentById): groups = relationship('GroupRoleAssociation', back_populates='role') library_actions = relationship('LibraryPermissions', back_populates='role') library_folder_actions = relationship('LibraryFolderPermissions', back_populates='role') + library_dataset_actions = relationship('LibraryDatasetPermissions', back_populates='role') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'name', 'description', 'type'] @@ -2602,7 +2603,18 @@ class LibraryFolderPermissions(Base, RepresentById): self.role = role -class LibraryDatasetPermissions(RepresentById): +class LibraryDatasetPermissions(Base, RepresentById): + __tablename__ = 'library_dataset_permissions' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + action = Column('action', TEXT) + library_dataset_id = Column('library_dataset_id', Integer, ForeignKey('library_dataset.id'), nullable=True, index=True) + role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + library_dataset = relationship('LibraryDataset', back_populates='actions') + role = relationship('Role', back_populates='library_dataset_actions') + def __init__(self, action, library_item, role): self.action = action if isinstance(library_item, LibraryDataset): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 9774b44e1e7..436e0e3329d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -199,15 +199,6 @@ model.UserRoleAssociation.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now)) -model.LibraryDatasetPermissions.table = Table( - "library_dataset_permissions", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("action", TEXT), - Column("library_dataset_id", Integer, ForeignKey("library_dataset.id"), nullable=True, index=True), - Column("role_id", Integer, ForeignKey("role.id"), index=True)) - model.LibraryDatasetDatasetAssociationPermissions.table = Table( "library_dataset_dataset_association_permissions", metadata, Column("id", Integer, primary_key=True), @@ -1294,11 +1285,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.LibraryDatasetPermissions, model.LibraryDatasetPermissions.table, properties=dict( - library_dataset=relation(model.LibraryDataset, backref="actions"), - role=relation(model.Role, backref="library_dataset_actions") -)) - mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociationPermissions, model.LibraryDatasetDatasetAssociationPermissions.table, properties=dict( library_dataset_dataset_association=relation(model.LibraryDatasetDatasetAssociation, backref="actions"), role=relation(model.Role, backref="library_dataset_dataset_actions") @@ -1384,7 +1370,8 @@ mapper_registry.map_imperatively(model.LibraryDataset, model.LibraryDataset.tabl == model.LibraryDatasetDatasetAssociation.table.c.id)) ), viewonly=True, - uselist=True) + uselist=True), + actions=relation(model.LibraryDatasetPermissions, back_populates='library_dataset'), )) mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssociation.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index c3725e824d8..ee38be26e90 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -741,6 +741,31 @@ def test_LibraryDatasetDatasetAssociationTagAssociation( assert stored_obj.user.id == user.id +def test_LibraryDatasetPermissions(model, session, library_dataset, role): + cls = model.LibraryDatasetPermissions + assert cls.__tablename__ == 'library_dataset_permissions' + with dbcleanup(session, cls): + action = 'a' + obj = cls(action, library_dataset, role) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.action == action + assert stored_obj.library_dataset_id == library_dataset.id + assert stored_obj.role_id == role.id + # test mapped relationships + assert stored_obj.library_dataset.id == library_dataset.id + assert stored_obj.role.id == role.id + + def test_LibraryFolderPermissions(model, session, library_folder, role): cls = model.LibraryFolderPermissions assert cls.__tablename__ == 'library_folder_permissions' @@ -1116,6 +1141,7 @@ def test_Role( group_role_association, library_permission, library_folder_permission, + library_dataset_permission, ): cls = model.Role assert cls.__tablename__ == 'role' @@ -1129,6 +1155,7 @@ def test_Role( obj.dataset_actions.append(dataset_permission) obj.library_actions.append(library_permission) obj.library_folder_actions.append(library_folder_permission) + obj.library_dataset_actions.append(library_dataset_permission) obj.groups.append(group_role_association) obj_id = persist(session, obj) @@ -1146,6 +1173,7 @@ def test_Role( assert stored_obj.groups == [group_role_association] assert stored_obj.library_actions == [library_permission] assert stored_obj.library_folder_actions == [library_folder_permission] + assert stored_obj.library_dataset_actions == [library_dataset_permission] def test_StoredWorkflowAnnotationAssociation(model, session, stored_workflow, user): @@ -1727,6 +1755,12 @@ def library_folder(model, session): yield from dbcleanup_wrapper(session, lf) +@pytest.fixture +def library_dataset(model, session): + ld = model.LibraryDataset() + yield from dbcleanup_wrapper(session, ld) + + @pytest.fixture def library_dataset_collection_association(model, session): ldca = model.LibraryDatasetCollectionAssociation() @@ -1751,6 +1785,12 @@ def library_dataset_dataset_association_tag_association(model, session): yield from dbcleanup_wrapper(session, lddata) +@pytest.fixture +def library_dataset_permission(model, session, library_dataset, role): + ldp = model.LibraryDatasetPermissions('a', library_dataset, role) + yield from dbcleanup_wrapper(session, ldp) + + @pytest.fixture def library_folder_permission(model, session, library_folder, role): lfp = model.LibraryFolderPermissions('a', library_folder, role) From 050f52d4148399810bcd819ede989864a1951ac3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 11:49:48 -0400 Subject: [PATCH 084/225] Map LDDAPermissions declaratively --- lib/galaxy/model/__init__.py | 19 +++++++++++++++++-- lib/galaxy/model/mapping.py | 21 ++++----------------- test/unit/model/test_model.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ff5eb228e5e..7017fbe8cda 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2415,6 +2415,8 @@ class Role(Base, Dictifiable, RepresentById): library_actions = relationship('LibraryPermissions', back_populates='role') library_folder_actions = relationship('LibraryFolderPermissions', back_populates='role') library_dataset_actions = relationship('LibraryDatasetPermissions', back_populates='role') + library_dataset_dataset_actions = relationship( + 'LibraryDatasetDatasetAssociationPermissions', back_populates='role') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'name', 'description', 'type'] @@ -2610,7 +2612,8 @@ class LibraryDatasetPermissions(Base, RepresentById): create_time = Column('create_time', DateTime, default=now) update_time = Column('update_time', DateTime, default=now, onupdate=now) action = Column('action', TEXT) - library_dataset_id = Column('library_dataset_id', Integer, ForeignKey('library_dataset.id'), nullable=True, index=True) + library_dataset_id = Column('library_dataset_id', Integer, + ForeignKey('library_dataset.id'), nullable=True, index=True) role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) library_dataset = relationship('LibraryDataset', back_populates='actions') role = relationship('Role', back_populates='library_dataset_actions') @@ -2624,7 +2627,19 @@ class LibraryDatasetPermissions(Base, RepresentById): self.role = role -class LibraryDatasetDatasetAssociationPermissions(RepresentById): +class LibraryDatasetDatasetAssociationPermissions(Base, RepresentById): + __tablename__ = 'library_dataset_dataset_association_permissions' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + action = Column('action', TEXT) + library_dataset_dataset_association_id = Column('library_dataset_dataset_association_id', + Integer, ForeignKey('library_dataset_dataset_association.id'), nullable=True, index=True) + role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', back_populates='actions') + role = relationship('Role', back_populates='library_dataset_dataset_actions') + def __init__(self, action, library_item, role): self.action = action if isinstance(library_item, LibraryDatasetDatasetAssociation): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 436e0e3329d..ce7be873531 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -199,17 +199,6 @@ model.UserRoleAssociation.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now)) -model.LibraryDatasetDatasetAssociationPermissions.table = Table( - "library_dataset_dataset_association_permissions", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("action", TEXT), - Column("library_dataset_dataset_association_id", Integer, - ForeignKey("library_dataset_dataset_association.id"), - nullable=True, index=True), - Column("role_id", Integer, ForeignKey("role.id"), index=True)) - model.DefaultHistoryPermissions.table = Table( "default_history_permissions", metadata, Column("id", Integer, primary_key=True), @@ -1285,11 +1274,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociationPermissions, model.LibraryDatasetDatasetAssociationPermissions.table, properties=dict( - library_dataset_dataset_association=relation(model.LibraryDatasetDatasetAssociation, backref="actions"), - role=relation(model.Role, backref="library_dataset_dataset_actions") -)) - mapper_registry.map_imperatively(model.ExtendedMetadata, model.ExtendedMetadata.table, properties=dict( children=relation(model.ExtendedMetadataIndex, backref='extended_metadata') )) @@ -1402,7 +1386,10 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.L extended_metadata=relation(model.ExtendedMetadata, primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.extended_metadata_id == model.ExtendedMetadata.table.c.id) ), - _metadata=deferred(model.LibraryDatasetDatasetAssociation.table.c._metadata) + _metadata=deferred(model.LibraryDatasetDatasetAssociation.table.c._metadata), + actions=relation( + model.LibraryDatasetDatasetAssociationPermissions, + back_populates='library_dataset_dataset_association'), )) mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, model.LibraryDatasetDatasetInfoAssociation.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index ee38be26e90..b1b112f6935 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -715,6 +715,32 @@ def test_LibraryDatasetCollectionRatingAssociation( assert stored_obj.user.id == user.id +def test_LibraryDatasetDatasetAssociationPermissions( + model, session, library_dataset_dataset_association, role): + cls = model.LibraryDatasetDatasetAssociationPermissions + assert cls.__tablename__ == 'library_dataset_dataset_association_permissions' + with dbcleanup(session, cls): + action = 'a' + obj = cls(action, library_dataset_dataset_association, role) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.action == action + assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert stored_obj.role_id == role.id + # test mapped relationships + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert stored_obj.role.id == role.id + + def test_LibraryDatasetDatasetAssociationTagAssociation( model, session, library_dataset_dataset_association, tag, user): cls = model.LibraryDatasetDatasetAssociationTagAssociation @@ -1142,6 +1168,7 @@ def test_Role( library_permission, library_folder_permission, library_dataset_permission, + library_dataset_dataset_association_permission, ): cls = model.Role assert cls.__tablename__ == 'role' @@ -1156,6 +1183,7 @@ def test_Role( obj.library_actions.append(library_permission) obj.library_folder_actions.append(library_folder_permission) obj.library_dataset_actions.append(library_dataset_permission) + obj.library_dataset_dataset_actions.append(library_dataset_dataset_association_permission) obj.groups.append(group_role_association) obj_id = persist(session, obj) @@ -1174,6 +1202,7 @@ def test_Role( assert stored_obj.library_actions == [library_permission] assert stored_obj.library_folder_actions == [library_folder_permission] assert stored_obj.library_dataset_actions == [library_dataset_permission] + assert stored_obj.library_dataset_dataset_actions == [library_dataset_dataset_association_permission] def test_StoredWorkflowAnnotationAssociation(model, session, stored_workflow, user): @@ -1791,6 +1820,12 @@ def library_dataset_permission(model, session, library_dataset, role): yield from dbcleanup_wrapper(session, ldp) +@pytest.fixture +def library_dataset_dataset_association_permission(model, session, library_dataset_dataset_association, role): + lddp = model.LibraryDatasetDatasetAssociationPermissions('a', library_dataset_dataset_association, role) + yield from dbcleanup_wrapper(session, lddp) + + @pytest.fixture def library_folder_permission(model, session, library_folder, role): lfp = model.LibraryFolderPermissions('a', library_folder, role) From 1a1b690718e5989681f77aadc8a91d6a6225490c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 12:07:59 -0400 Subject: [PATCH 085/225] Map DefaultHistoryPermissions declaratively --- lib/galaxy/model/__init__.py | 11 ++++++++++- lib/galaxy/model/mapping.py | 13 +------------ test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7017fbe8cda..80b9ec53980 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2665,7 +2665,16 @@ class DefaultUserPermissions(Base, RepresentById): self.role = role -class DefaultHistoryPermissions(RepresentById): +class DefaultHistoryPermissions(Base, RepresentById): + __tablename__ = 'default_history_permissions' + + id = Column('id', Integer, primary_key=True) + history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) + action = Column('action', TEXT) + role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + history = relationship('History', back_populates='default_permissions') + role = relationship('Role') + def __init__(self, history, action, role): self.history = history self.action = action diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index ce7be873531..f67506bb584 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -199,13 +199,6 @@ model.UserRoleAssociation.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now)) -model.DefaultHistoryPermissions.table = Table( - "default_history_permissions", metadata, - Column("id", Integer, primary_key=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("action", TEXT), - Column("role_id", Integer, ForeignKey("role.id"), index=True)) - model.LibraryDataset.table = Table( "library_dataset", metadata, Column("id", Integer, primary_key=True), @@ -1194,6 +1187,7 @@ mapper_registry.map_imperatively(model.History, model.History.table, properties= update_time=column_property( select(func.max(model.HistoryAudit.update_time)).where(model.HistoryAudit.history_id == model.History.table.c.id).scalar_subquery(), ), + default_permissions=relation(model.DefaultHistoryPermissions, back_populates='history'), )) # Set up proxy so that @@ -1255,11 +1249,6 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( # .preferences[pref_name] = pref_value model.User.preferences = association_proxy('_preferences', 'value', creator=model.UserPreference) # type: ignore -mapper_registry.map_imperatively(model.DefaultHistoryPermissions, model.DefaultHistoryPermissions.table, properties=dict( - history=relation(model.History, backref="default_permissions"), - role=relation(model.Role) -)) - mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssociation.table, properties=dict( user=relation(model.User, backref="roles"), role=relation(model.Role, backref="users"), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index b1b112f6935..4caf9838ab2 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -181,6 +181,25 @@ def test_DatasetPermissions(model, session, dataset, role): assert stored_obj.role == role +def test_DefaultHistoryPermissions(model, session, history, role): + cls = model.DefaultHistoryPermissions + assert cls.__tablename__ == 'default_history_permissions' + with dbcleanup(session, cls): + action = 'a' + obj = cls(history, action, role) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.action == action + assert stored_obj.history_id == history.id + assert stored_obj.role_id == role.id + # test mapped relationships + assert stored_obj.history.id == history.id + assert stored_obj.role == role + + def test_DefaultQuotaAssociation(model, session, quota): cls = model.DefaultQuotaAssociation assert cls.__tablename__ == 'default_quota_association' From 18a51305f3fa0cf62dd2605edd735806136dba91 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 12:41:00 -0400 Subject: [PATCH 086/225] Map JobToInputDatasetAssociation declaratively --- lib/galaxy/model/__init__.py | 13 ++++++++++++- lib/galaxy/model/mapping.py | 19 +++---------------- test/unit/model/test_model.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 80b9ec53980..3533b494022 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1562,7 +1562,18 @@ class JobParameter(RepresentById): return JobParameter(name=self.name, value=self.value) -class JobToInputDatasetAssociation(RepresentById): +class JobToInputDatasetAssociation(Base, RepresentById): + __tablename__ = 'job_to_input_dataset' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + dataset_id = Column('dataset_id', Integer, + ForeignKey('history_dataset_association.id'), index=True) + dataset_version = Column('dataset_version', Integer) + name = Column('name', String(255)) + dataset = relationship('HistoryDatasetAssociation', lazy=False, back_populates='dependent_jobs') + job = relationship('Job', back_populates='input_datasets') + def __init__(self, name, dataset): self.name = name self.dataset = dataset diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f67506bb584..7e82ef3c74a 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -359,14 +359,6 @@ model.JobParameter.table = Table( Column("name", String(255)), Column("value", TEXT)) -model.JobToInputDatasetAssociation.table = Table( - "job_to_input_dataset", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True), - Column("dataset_version", Integer), - Column("name", String(255))) - model.JobToOutputDatasetAssociation.table = Table( "job_to_output_dataset", metadata, Column("id", Integer, primary_key=True), @@ -1072,7 +1064,8 @@ simple_mapping(model.HistoryDatasetAssociation, == model.HistoryDatasetCollectionAssociation.table.c.id), uselist=False, backref="hidden_dataset_instances"), - _metadata=deferred(model.HistoryDatasetAssociation.table.c._metadata) + _metadata=deferred(model.HistoryDatasetAssociation.table.c._metadata), + dependent_jobs=relation(model.JobToInputDatasetAssociation, back_populates='dataset'), ) simple_mapping(model.Dataset, @@ -1395,12 +1388,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -mapper_registry.map_imperatively(model.JobToInputDatasetAssociation, model.JobToInputDatasetAssociation.table, properties=dict( - dataset=relation(model.HistoryDatasetAssociation, - lazy=False, - backref="dependent_jobs") -)) - mapper_registry.map_imperatively(model.JobToOutputDatasetAssociation, model.JobToOutputDatasetAssociation.table, properties=dict( job=relation(model.Job, backref="output_datasets"), @@ -1899,7 +1886,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( history=relation(model.History, backref="jobs"), library_folder=relation(model.LibraryFolder, lazy=True), parameters=relation(model.JobParameter, lazy=True), - input_datasets=relation(model.JobToInputDatasetAssociation, backref="job"), + input_datasets=relation(model.JobToInputDatasetAssociation, back_populates="job"), input_dataset_collections=relation(model.JobToInputDatasetCollectionAssociation, backref="job", lazy=True), input_dataset_collection_elements=relation(model.JobToInputDatasetCollectionElementAssociation, backref="job", lazy=True), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 4caf9838ab2..e75eb647765 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -635,6 +635,30 @@ def test_HistoryTagAssociation(model, session, history, tag, user): assert stored_obj.user.id == user.id +def test_JobToInputDatasetAssociation(model, session, history_dataset_association, job): + cls = model.JobToInputDatasetAssociation + assert cls.__tablename__ == 'job_to_input_dataset' + + with dbcleanup(session, cls): + name, dataset_version = 'a', 9 + obj = cls(name, history_dataset_association) + obj.history_dataset_association_id = history_dataset_association.id + obj.job = job + obj.dataset_version = dataset_version + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_id == history_dataset_association.id + assert stored_obj.dataset_version == dataset_version + assert stored_obj.name == name + # test mapped relationships + assert stored_obj.job.id == job.id + assert stored_obj.dataset.id == history_dataset_association.id + + def test_Library(model, session, library_folder, library_permission): cls = model.Library assert cls.__tablename__ == 'library' @@ -1791,6 +1815,12 @@ def history_tag_association(model, session): yield from dbcleanup_wrapper(session, hta) +@pytest.fixture +def job(model, session): + j = model.Job() + yield from dbcleanup_wrapper(session, j) + + @pytest.fixture def library(model, session): lb = model.Library() From 9e3f5e6592f6559e3b147808cdb64a363df84a97 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 14:18:28 -0400 Subject: [PATCH 087/225] Map JtIDCA declaratively --- lib/galaxy/model/__init__.py | 13 ++++++++++++- lib/galaxy/model/mapping.py | 14 +------------- test/unit/model/test_model.py | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3533b494022..dfaf1498505 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1586,7 +1586,18 @@ class JobToOutputDatasetAssociation(RepresentById): self.dataset = dataset -class JobToInputDatasetCollectionAssociation(RepresentById): +class JobToInputDatasetCollectionAssociation(Base, RepresentById): + __tablename__ = 'job_to_input_dataset_collection' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + dataset_collection_id = Column('dataset_collection_id', Integer, + ForeignKey('history_dataset_collection_association.id'), index=True) + dataset_version = Column('dataset_version', Integer) + name = Column('name', String(255)) + dataset_collection = relationship('HistoryDatasetCollectionAssociation', lazy=False) + job = relationship('Job', back_populates='input_dataset_collections') + def __init__(self, name, dataset_collection): self.name = name self.dataset_collection = dataset_collection diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 7e82ef3c74a..ddcca308955 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -366,13 +366,6 @@ model.JobToOutputDatasetAssociation.table = Table( Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True), Column("name", String(255))) -model.JobToInputDatasetCollectionAssociation.table = Table( - "job_to_input_dataset_collection", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("dataset_collection_id", Integer, ForeignKey("history_dataset_collection_association.id"), index=True), - Column("name", Unicode(255))) - model.JobToInputDatasetCollectionElementAssociation.table = Table( "job_to_input_dataset_collection_element", metadata, Column("id", Integer, primary_key=True), @@ -1396,11 +1389,6 @@ mapper_registry.map_imperatively(model.JobToOutputDatasetAssociation, model.JobT backref="creating_job_associations") )) -mapper_registry.map_imperatively(model.JobToInputDatasetCollectionAssociation, model.JobToInputDatasetCollectionAssociation.table, properties=dict( - dataset_collection=relation(model.HistoryDatasetCollectionAssociation, - lazy=False) -)) - mapper_registry.map_imperatively(model.JobToInputDatasetCollectionElementAssociation, model.JobToInputDatasetCollectionElementAssociation.table, properties=dict( dataset_collection_element=relation(model.DatasetCollectionElement, lazy=False) @@ -1887,7 +1875,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( library_folder=relation(model.LibraryFolder, lazy=True), parameters=relation(model.JobParameter, lazy=True), input_datasets=relation(model.JobToInputDatasetAssociation, back_populates="job"), - input_dataset_collections=relation(model.JobToInputDatasetCollectionAssociation, backref="job", lazy=True), + input_dataset_collections=relation(model.JobToInputDatasetCollectionAssociation, back_populates="job", lazy=True), input_dataset_collection_elements=relation(model.JobToInputDatasetCollectionElementAssociation, backref="job", lazy=True), output_dataset_collection_instances=relation(model.JobToOutputDatasetCollectionAssociation, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index e75eb647765..ccda1fc9959 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -635,6 +635,28 @@ def test_HistoryTagAssociation(model, session, history, tag, user): assert stored_obj.user.id == user.id +def test_JobToInputDatasetCollectionAssociation( + model, session, history_dataset_collection_association, job): + cls = model.JobToInputDatasetCollectionAssociation + assert cls.__tablename__ == 'job_to_input_dataset_collection' + + with dbcleanup(session, cls): + name = 'a' + obj = cls(name, history_dataset_collection_association) + obj.job = job + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.name == name + # test mapped relationships + assert stored_obj.job.id == job.id + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id + + def test_JobToInputDatasetAssociation(model, session, history_dataset_association, job): cls = model.JobToInputDatasetAssociation assert cls.__tablename__ == 'job_to_input_dataset' From 0710af7e6c30d5cc556bfd6e832aa50be2f3368f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 16:12:21 -0400 Subject: [PATCH 088/225] Map JobToInputDatasetCollectionElementAssociation declaratively --- lib/galaxy/model/__init__.py | 11 ++++++++++- lib/galaxy/model/mapping.py | 14 +------------- test/unit/model/test_model.py | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index dfaf1498505..71106301fab 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1603,7 +1603,16 @@ class JobToInputDatasetCollectionAssociation(Base, RepresentById): self.dataset_collection = dataset_collection -class JobToInputDatasetCollectionElementAssociation(RepresentById): +class JobToInputDatasetCollectionElementAssociation(Base, RepresentById): + __tablename__ = 'job_to_input_dataset_collection_element' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + dataset_collection_element_id = Column('dataset_collection_element_id', Integer, ForeignKey('dataset_collection_element.id'), index=True) + name = Column('name', Unicode(255)) + dataset_collection_element = relationship('DatasetCollectionElement', lazy=False) + job = relationship('Job', back_populates='input_dataset_collection_elements') + def __init__(self, name, dataset_collection_element): self.name = name self.dataset_collection_element = dataset_collection_element diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index ddcca308955..e99ef99c682 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -366,13 +366,6 @@ model.JobToOutputDatasetAssociation.table = Table( Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True), Column("name", String(255))) -model.JobToInputDatasetCollectionElementAssociation.table = Table( - "job_to_input_dataset_collection_element", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("dataset_collection_element_id", Integer, ForeignKey("dataset_collection_element.id"), index=True), - Column("name", Unicode(255))) - model.JobToImplicitOutputDatasetCollectionAssociation.table = Table( "job_to_implicit_output_dataset_collection", metadata, Column("id", Integer, primary_key=True), @@ -1389,11 +1382,6 @@ mapper_registry.map_imperatively(model.JobToOutputDatasetAssociation, model.JobT backref="creating_job_associations") )) -mapper_registry.map_imperatively(model.JobToInputDatasetCollectionElementAssociation, model.JobToInputDatasetCollectionElementAssociation.table, properties=dict( - dataset_collection_element=relation(model.DatasetCollectionElement, - lazy=False) -)) - mapper_registry.map_imperatively(model.JobToOutputDatasetCollectionAssociation, model.JobToOutputDatasetCollectionAssociation.table, properties=dict( dataset_collection_instance=relation(model.HistoryDatasetCollectionAssociation, lazy=False, @@ -1877,7 +1865,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( input_datasets=relation(model.JobToInputDatasetAssociation, back_populates="job"), input_dataset_collections=relation(model.JobToInputDatasetCollectionAssociation, back_populates="job", lazy=True), input_dataset_collection_elements=relation(model.JobToInputDatasetCollectionElementAssociation, - backref="job", lazy=True), + back_populates="job", lazy=True), output_dataset_collection_instances=relation(model.JobToOutputDatasetCollectionAssociation, backref="job", lazy=True), output_dataset_collections=relation(model.JobToImplicitOutputDatasetCollectionAssociation, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index ccda1fc9959..6c088eae02e 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -657,6 +657,28 @@ def test_JobToInputDatasetCollectionAssociation( assert stored_obj.dataset_collection.id == history_dataset_collection_association.id +def test_JobToInputDatasetCollectionElementAssociation( + model, session, dataset_collection_element, job): + cls = model.JobToInputDatasetCollectionElementAssociation + assert cls.__tablename__ == 'job_to_input_dataset_collection_element' + + with dbcleanup(session, cls): + name = 'a' + obj = cls(name, dataset_collection_element) + obj.job = job + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_collection_element_id == dataset_collection_element.id + assert stored_obj.name == name + # test mapped relationships + assert stored_obj.job.id == job.id + assert stored_obj.dataset_collection_element.id == dataset_collection_element.id + + def test_JobToInputDatasetAssociation(model, session, history_dataset_association, job): cls = model.JobToInputDatasetAssociation assert cls.__tablename__ == 'job_to_input_dataset' @@ -1740,6 +1762,20 @@ def dataset(model, session): yield from dbcleanup_wrapper(session, d) +@pytest.fixture +def dataset_collection(model, session): + dc = model.DatasetCollection(collection_type='a') + yield from dbcleanup_wrapper(session, dc) + + +@pytest.fixture +def dataset_collection_element( + model, session, dataset_collection, history_dataset_association): + dce = model.DatasetCollectionElement( + collection=dataset_collection, element=history_dataset_association) + yield from dbcleanup_wrapper(session, dce) + + @pytest.fixture def dataset_permission(model, session, dataset): d = model.DatasetPermissions('a', dataset) From ceb97e58f08025149783d87818f20ad83e6a9dc5 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 16:27:29 -0400 Subject: [PATCH 089/225] Map JobToOutputDatasetCollectionAssociation declaratively --- lib/galaxy/model/__init__.py | 14 +++++++++++++- lib/galaxy/model/mapping.py | 24 +++++++----------------- test/unit/model/test_model.py | 22 ++++++++++++++++++++++ 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 71106301fab..188f70025cb 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1620,7 +1620,19 @@ class JobToInputDatasetCollectionElementAssociation(Base, RepresentById): # Many jobs may map to one HistoryDatasetCollection using these for a given # tool output (if mapping over an input collection). -class JobToOutputDatasetCollectionAssociation(RepresentById): +class JobToOutputDatasetCollectionAssociation(Base, RepresentById): + __tablename__ = 'job_to_output_dataset_collection' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + dataset_collection_id = Column('dataset_collection_id', Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + name = Column('name', Unicode(255)) + dataset_collection_instance = relationship( + 'HistoryDatasetCollectionAssociation', + lazy=False, + back_populates="output_dataset_collection_instances") + job = relationship('Job', back_populates='output_dataset_collection_instances') + def __init__(self, name, dataset_collection_instance): self.name = name self.dataset_collection_instance = dataset_collection_instance diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index e99ef99c682..84e791c2850 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -373,13 +373,6 @@ model.JobToImplicitOutputDatasetCollectionAssociation.table = Table( Column("dataset_collection_id", Integer, ForeignKey("dataset_collection.id"), index=True), Column("name", Unicode(255))) -model.JobToOutputDatasetCollectionAssociation.table = Table( - "job_to_output_dataset_collection", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("dataset_collection_id", Integer, ForeignKey("history_dataset_collection_association.id"), index=True), - Column("name", Unicode(255))) - model.JobToInputLibraryDatasetAssociation.table = Table( "job_to_input_library_dataset", metadata, Column("id", Integer, primary_key=True), @@ -1382,12 +1375,6 @@ mapper_registry.map_imperatively(model.JobToOutputDatasetAssociation, model.JobT backref="creating_job_associations") )) -mapper_registry.map_imperatively(model.JobToOutputDatasetCollectionAssociation, model.JobToOutputDatasetCollectionAssociation.table, properties=dict( - dataset_collection_instance=relation(model.HistoryDatasetCollectionAssociation, - lazy=False, - backref="output_dataset_collection_instances") -)) - mapper_registry.map_imperatively(model.JobToImplicitOutputDatasetCollectionAssociation, model.JobToImplicitOutputDatasetCollectionAssociation.table, properties=dict( dataset_collection=relation(model.DatasetCollection, backref="output_dataset_collections") @@ -1549,7 +1536,10 @@ simple_mapping(model.HistoryDatasetCollectionAssociation, back_populates="history_dataset_collection"), ratings=relation(model.HistoryDatasetCollectionRatingAssociation, order_by=model.HistoryDatasetCollectionRatingAssociation.id, - back_populates="dataset_collection") + back_populates="dataset_collection"), + output_dataset_collection_instances=relation( + model.JobToOutputDatasetCollectionAssociation, + back_populates='dataset_collection_instance'), ) simple_mapping(model.LibraryDatasetCollectionAssociation, @@ -1867,7 +1857,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( input_dataset_collection_elements=relation(model.JobToInputDatasetCollectionElementAssociation, back_populates="job", lazy=True), output_dataset_collection_instances=relation(model.JobToOutputDatasetCollectionAssociation, - backref="job", lazy=True), + back_populates="job", lazy=True), output_dataset_collections=relation(model.JobToImplicitOutputDatasetCollectionAssociation, backref="job", lazy=True), post_job_actions=relation(model.PostJobActionAssociation, backref="job", lazy=False), @@ -1886,8 +1876,8 @@ model.Job.any_output_dataset_deleted = column_property( # type: ignore ) model.Job.any_output_dataset_collection_instances_deleted = column_property( # type: ignore exists([model.HistoryDatasetCollectionAssociation.table.c.id], - and_(model.Job.table.c.id == model.JobToOutputDatasetCollectionAssociation.table.c.job_id, - model.HistoryDatasetCollectionAssociation.table.c.id == model.JobToOutputDatasetCollectionAssociation.table.c.dataset_collection_id, + and_(model.Job.table.c.id == model.JobToOutputDatasetCollectionAssociation.job_id, + model.HistoryDatasetCollectionAssociation.table.c.id == model.JobToOutputDatasetCollectionAssociation.dataset_collection_id, model.HistoryDatasetCollectionAssociation.table.c.deleted == true()) ) ) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 6c088eae02e..bcdf383efc3 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -703,6 +703,28 @@ def test_JobToInputDatasetAssociation(model, session, history_dataset_associatio assert stored_obj.dataset.id == history_dataset_association.id +def test_JobToOutputDatasetCollectionAssociation( + model, session, history_dataset_collection_association, job): + cls = model.JobToOutputDatasetCollectionAssociation + assert cls.__tablename__ == 'job_to_output_dataset_collection' + + with dbcleanup(session, cls): + name = 'a' + obj = cls(name, history_dataset_collection_association) + obj.job = job + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.name == name + # test mapped relationships + assert stored_obj.job.id == job.id + assert stored_obj.dataset_collection_instance.id == history_dataset_collection_association.id + + def test_Library(model, session, library_folder, library_permission): cls = model.Library assert cls.__tablename__ == 'library' From 3b6369eb6852f33647f2c6beaf8580f6e57318d7 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 16:37:54 -0400 Subject: [PATCH 090/225] Map JobToOutputDatasetAssociation declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 24 ++++++------------------ test/unit/model/test_model.py | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 188f70025cb..e094f121350 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1580,7 +1580,17 @@ class JobToInputDatasetAssociation(Base, RepresentById): self.dataset_version = 0 # We start with version 0 and update once the job is ready -class JobToOutputDatasetAssociation(RepresentById): +class JobToOutputDatasetAssociation(Base, RepresentById): + __tablename__ = 'job_to_output_dataset' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + dataset_id = Column('dataset_id', Integer, + ForeignKey('history_dataset_association.id'), index=True) + name = Column('name', String(255)) + dataset = relationship('HistoryDatasetAssociation', lazy=False, back_populates='creating_job_associations') + job = relationship('Job', back_populates='output_datasets') + def __init__(self, name, dataset): self.name = name self.dataset = dataset diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 84e791c2850..f132ec3bb97 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -359,13 +359,6 @@ model.JobParameter.table = Table( Column("name", String(255)), Column("value", TEXT)) -model.JobToOutputDatasetAssociation.table = Table( - "job_to_output_dataset", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True), - Column("name", String(255))) - model.JobToImplicitOutputDatasetCollectionAssociation.table = Table( "job_to_implicit_output_dataset_collection", metadata, Column("id", Integer, primary_key=True), @@ -1045,6 +1038,8 @@ simple_mapping(model.HistoryDatasetAssociation, backref="hidden_dataset_instances"), _metadata=deferred(model.HistoryDatasetAssociation.table.c._metadata), dependent_jobs=relation(model.JobToInputDatasetAssociation, back_populates='dataset'), + creating_job_associations=relation( + model.JobToOutputDatasetAssociation, back_populates='dataset'), ) simple_mapping(model.Dataset, @@ -1367,14 +1362,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -mapper_registry.map_imperatively(model.JobToOutputDatasetAssociation, model.JobToOutputDatasetAssociation.table, properties=dict( - job=relation(model.Job, - backref="output_datasets"), - dataset=relation(model.HistoryDatasetAssociation, - lazy=False, - backref="creating_job_associations") -)) - mapper_registry.map_imperatively(model.JobToImplicitOutputDatasetCollectionAssociation, model.JobToImplicitOutputDatasetCollectionAssociation.table, properties=dict( dataset_collection=relation(model.DatasetCollection, backref="output_dataset_collections") @@ -1865,12 +1852,13 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( output_library_datasets=relation(model.JobToOutputLibraryDatasetAssociation, backref="job", lazy=True), external_output_metadata=relation(model.JobExternalOutputMetadata, lazy=True, backref='job'), - tasks=relation(model.Task, backref='job') + tasks=relation(model.Task, backref='job'), + output_datasets=relation(model.JobToOutputDatasetAssociation, back_populates='job'), )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], - and_(model.Job.table.c.id == model.JobToOutputDatasetAssociation.table.c.job_id, - model.HistoryDatasetAssociation.table.c.id == model.JobToOutputDatasetAssociation.table.c.dataset_id, + and_(model.Job.table.c.id == model.JobToOutputDatasetAssociation.job_id, + model.HistoryDatasetAssociation.table.c.id == model.JobToOutputDatasetAssociation.dataset_id, model.HistoryDatasetAssociation.table.c.deleted == true()) ) ) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index bcdf383efc3..d3a4b08f34c 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -703,6 +703,27 @@ def test_JobToInputDatasetAssociation(model, session, history_dataset_associatio assert stored_obj.dataset.id == history_dataset_association.id +def test_JobToOutputDatasetAssociation(model, session, history_dataset_association, job): + cls = model.JobToOutputDatasetAssociation + assert cls.__tablename__ == 'job_to_output_dataset' + + with dbcleanup(session, cls): + name = 'a' + obj = cls(name, history_dataset_association) + obj.job = job + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_id == history_dataset_association.id + assert stored_obj.name == name + # test mapped relationships + assert stored_obj.job.id == job.id + assert stored_obj.dataset.id == history_dataset_association.id + + def test_JobToOutputDatasetCollectionAssociation( model, session, history_dataset_collection_association, job): cls = model.JobToOutputDatasetCollectionAssociation From ccd8b8ef08df50f2a72b8f64e5e7eebac9d815e6 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 16:52:44 -0400 Subject: [PATCH 091/225] Map model.JobToImplicitOutputDatasetCollectionAssociation declaratively --- lib/galaxy/model/__init__.py | 13 ++++++++++++- lib/galaxy/model/mapping.py | 18 ++++-------------- test/unit/model/test_model.py | 22 ++++++++++++++++++++++ 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e094f121350..5854e53282b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1651,7 +1651,18 @@ class JobToOutputDatasetCollectionAssociation(Base, RepresentById): # A DatasetCollection will be mapped to at most one job per tool output # using these. (You can think of many of these models as going into the # creation of a JobToOutputDatasetCollectionAssociation.) -class JobToImplicitOutputDatasetCollectionAssociation(RepresentById): +class JobToImplicitOutputDatasetCollectionAssociation(Base, RepresentById): + __tablename__ = 'job_to_implicit_output_dataset_collection' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + dataset_collection_id = Column( + 'dataset_collection_id', Integer, ForeignKey('dataset_collection.id'), index=True) + name = Column('name', Unicode(255)) + dataset_collection = relationship( + 'DatasetCollection', back_populates="output_dataset_collections") + job = relationship('Job', back_populates='output_dataset_collections') + def __init__(self, name, dataset_collection): self.name = name self.dataset_collection = dataset_collection diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index f132ec3bb97..6730c512d77 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -359,13 +359,6 @@ model.JobParameter.table = Table( Column("name", String(255)), Column("value", TEXT)) -model.JobToImplicitOutputDatasetCollectionAssociation.table = Table( - "job_to_implicit_output_dataset_collection", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("dataset_collection_id", Integer, ForeignKey("dataset_collection.id"), index=True), - Column("name", Unicode(255))) - model.JobToInputLibraryDatasetAssociation.table = Table( "job_to_input_library_dataset", metadata, Column("id", Integer, primary_key=True), @@ -1362,11 +1355,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -mapper_registry.map_imperatively(model.JobToImplicitOutputDatasetCollectionAssociation, model.JobToImplicitOutputDatasetCollectionAssociation.table, properties=dict( - dataset_collection=relation(model.DatasetCollection, - backref="output_dataset_collections") -)) - mapper_registry.map_imperatively(model.JobToInputLibraryDatasetAssociation, model.JobToInputLibraryDatasetAssociation.table, properties=dict( dataset=relation(model.LibraryDatasetDatasetAssociation, lazy=False, @@ -1482,7 +1470,9 @@ simple_mapping(model.DatasetCollection, primaryjoin=(model.DatasetCollection.table.c.id == model.DatasetCollectionElement.table.c.dataset_collection_id), remote_side=[model.DatasetCollectionElement.table.c.dataset_collection_id], backref="collection", - order_by=model.DatasetCollectionElement.table.c.element_index) + order_by=model.DatasetCollectionElement.table.c.element_index), + output_dataset_collections=relation( + model.JobToImplicitOutputDatasetCollectionAssociation, back_populates='dataset_collection'), ) simple_mapping(model.HistoryDatasetCollectionAssociation, @@ -1846,7 +1836,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( output_dataset_collection_instances=relation(model.JobToOutputDatasetCollectionAssociation, back_populates="job", lazy=True), output_dataset_collections=relation(model.JobToImplicitOutputDatasetCollectionAssociation, - backref="job", lazy=True), + back_populates="job", lazy=True), post_job_actions=relation(model.PostJobActionAssociation, backref="job", lazy=False), input_library_datasets=relation(model.JobToInputLibraryDatasetAssociation, backref="job"), output_library_datasets=relation(model.JobToOutputLibraryDatasetAssociation, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index d3a4b08f34c..33e5e315b47 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -635,6 +635,28 @@ def test_HistoryTagAssociation(model, session, history, tag, user): assert stored_obj.user.id == user.id +def test_JobToImplicitOutputDatasetCollectionAssociation( + model, session, dataset_collection, job): + cls = model.JobToImplicitOutputDatasetCollectionAssociation + assert cls.__tablename__ == 'job_to_implicit_output_dataset_collection' + + with dbcleanup(session, cls): + name = 'a' + obj = cls(name, dataset_collection) + obj.job = job + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_collection_id == dataset_collection.id + assert stored_obj.name == name + # test mapped relationships + assert stored_obj.job.id == job.id + assert stored_obj.dataset_collection.id == dataset_collection.id + + def test_JobToInputDatasetCollectionAssociation( model, session, history_dataset_collection_association, job): cls = model.JobToInputDatasetCollectionAssociation From fae7aebe7e840bb416b545e52d36c076e77cd341 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 17:07:13 -0400 Subject: [PATCH 092/225] Fix tests for consistent testing of relationships/fkeys --- test/unit/model/test_model.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 33e5e315b47..2c4a872b2a0 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -504,7 +504,7 @@ def test_HistoryDatasetAssociationTagAssociation( user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) obj.user_value = user_value - obj.history_dataset_association_id = history_dataset_association.id + obj.history_dataset_association = history_dataset_association obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) @@ -572,7 +572,7 @@ def test_HistoryDatasetCollectionTagAssociation(model, session, history_dataset_ user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) obj.user_value = user_value - obj.history_dataset_collection_id = history_dataset_collection_association.id + obj.dataset_collection = history_dataset_collection_association obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) @@ -617,7 +617,7 @@ def test_HistoryTagAssociation(model, session, history, tag, user): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) obj.user_value = user_value - obj.history_id = history.id + obj.history = history obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) @@ -708,7 +708,6 @@ def test_JobToInputDatasetAssociation(model, session, history_dataset_associatio with dbcleanup(session, cls): name, dataset_version = 'a', 9 obj = cls(name, history_dataset_association) - obj.history_dataset_association_id = history_dataset_association.id obj.job = job obj.dataset_version = dataset_version obj_id = persist(session, obj) @@ -829,7 +828,7 @@ def test_LibraryDatasetCollectionTagAssociation(model, session, library_dataset_ user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) obj.user_value = user_value - obj.library_dataset_collection_id = library_dataset_collection_association.id + obj.dataset_collection = library_dataset_collection_association obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) @@ -901,7 +900,7 @@ def test_LibraryDatasetDatasetAssociationTagAssociation( user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) obj.user_value = user_value - obj.library_dataset_dataset_association_id = library_dataset_dataset_association.id + obj.library_dataset_dataset_association = library_dataset_dataset_association obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) @@ -1145,7 +1144,7 @@ def test_PageTagAssociation(model, session, page, tag, user): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) obj.user_value = user_value - obj.page_id = page.id + obj.page = page obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) @@ -1406,7 +1405,7 @@ def test_StoredWorkflowTagAssociation(model, session, stored_workflow, tag, user user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) obj.user_value = user_value - obj.stored_workflow_id = stored_workflow.id + obj.stored_workflow = stored_workflow obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) @@ -1759,7 +1758,7 @@ def test_WorkflowStepTagAssociation(model, session, workflow_step, tag, user): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) obj.user_value = user_value - obj.workflow_step_id = workflow_step.id + obj.workflow_step = workflow_step obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) @@ -1784,7 +1783,7 @@ def test_VisualizationTagAssociation(model, session, visualization, tag, user): user_tname, value, user_value = 'a', 'b', 'c' obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) obj.user_value = user_value - obj.visualization_id = visualization.id + obj.visualization = visualization obj_id = persist(session, obj) stored_obj = get_stored_obj(session, cls, obj_id) From 3e64819e97756adfac47e5e57d0dfbc1bee3041c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 17:15:01 -0400 Subject: [PATCH 093/225] Map JobToInputLibraryDatasetAssociation declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 17 +++-------------- test/unit/model/test_model.py | 22 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 5854e53282b..e98eb29dbf6 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1668,7 +1668,17 @@ class JobToImplicitOutputDatasetCollectionAssociation(Base, RepresentById): self.dataset_collection = dataset_collection -class JobToInputLibraryDatasetAssociation(RepresentById): +class JobToInputLibraryDatasetAssociation(Base, RepresentById): + __tablename__ = 'job_to_input_library_dataset' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + ldda_id = Column('ldda_id', Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + name = Column('name', Unicode(255)) + job = relationship('Job', back_populates='input_library_datasets') + dataset = relationship( + 'LibraryDatasetDatasetAssociation', lazy=False, back_populates='dependent_jobs') + def __init__(self, name, dataset): self.name = name self.dataset = dataset diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6730c512d77..419327a3f1d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -359,13 +359,6 @@ model.JobParameter.table = Table( Column("name", String(255)), Column("value", TEXT)) -model.JobToInputLibraryDatasetAssociation.table = Table( - "job_to_input_library_dataset", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("ldda_id", Integer, ForeignKey("library_dataset_dataset_association.id"), index=True), - Column("name", String(255))) - model.JobToOutputLibraryDatasetAssociation.table = Table( "job_to_output_library_dataset", metadata, Column("id", Integer, primary_key=True), @@ -1339,6 +1332,8 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.L actions=relation( model.LibraryDatasetDatasetAssociationPermissions, back_populates='library_dataset_dataset_association'), + dependent_jobs=relation( + model.JobToInputLibraryDatasetAssociation, back_populates='dataset'), )) mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, model.LibraryDatasetDatasetInfoAssociation.table, properties=dict( @@ -1355,12 +1350,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -mapper_registry.map_imperatively(model.JobToInputLibraryDatasetAssociation, model.JobToInputLibraryDatasetAssociation.table, properties=dict( - dataset=relation(model.LibraryDatasetDatasetAssociation, - lazy=False, - backref="dependent_jobs") -)) - mapper_registry.map_imperatively(model.JobToOutputLibraryDatasetAssociation, model.JobToOutputLibraryDatasetAssociation.table, properties=dict( dataset=relation(model.LibraryDatasetDatasetAssociation, lazy=False, @@ -1838,7 +1827,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( output_dataset_collections=relation(model.JobToImplicitOutputDatasetCollectionAssociation, back_populates="job", lazy=True), post_job_actions=relation(model.PostJobActionAssociation, backref="job", lazy=False), - input_library_datasets=relation(model.JobToInputLibraryDatasetAssociation, backref="job"), + input_library_datasets=relation(model.JobToInputLibraryDatasetAssociation, back_populates="job"), output_library_datasets=relation(model.JobToOutputLibraryDatasetAssociation, backref="job", lazy=True), external_output_metadata=relation(model.JobExternalOutputMetadata, lazy=True, backref='job'), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 2c4a872b2a0..12088c5932c 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -724,6 +724,28 @@ def test_JobToInputDatasetAssociation(model, session, history_dataset_associatio assert stored_obj.dataset.id == history_dataset_association.id +def test_JobToInputLibraryDatasetAssociation( + model, session, library_dataset_dataset_association, job): + cls = model.JobToInputLibraryDatasetAssociation + assert cls.__tablename__ == 'job_to_input_library_dataset' + + with dbcleanup(session, cls): + name = 'a' + obj = cls(name, library_dataset_dataset_association) + obj.job = job + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.ldda_id == library_dataset_dataset_association.id + assert stored_obj.name == name + # test mapped relationships + assert stored_obj.job.id == job.id + assert stored_obj.dataset.id == library_dataset_dataset_association.id + + def test_JobToOutputDatasetAssociation(model, session, history_dataset_association, job): cls = model.JobToOutputDatasetAssociation assert cls.__tablename__ == 'job_to_output_dataset' From d89b4b73165ccd86bdb4ba62161e68339c3d5485 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 1 Jul 2021 18:00:39 -0400 Subject: [PATCH 094/225] Map model.JobToOutputLibraryDatasetAssociation declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 17 +++-------------- test/unit/model/test_model.py | 21 +++++++++++++++++++++ 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e98eb29dbf6..445a4086abc 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1684,7 +1684,17 @@ class JobToInputLibraryDatasetAssociation(Base, RepresentById): self.dataset = dataset -class JobToOutputLibraryDatasetAssociation(RepresentById): +class JobToOutputLibraryDatasetAssociation(Base, RepresentById): + __tablename__ = 'job_to_output_library_dataset' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + ldda_id = Column('ldda_id', Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + name = Column('name', Unicode(255)) + job = relationship('Job', back_populates='output_library_datasets') + dataset = relationship( + 'LibraryDatasetDatasetAssociation', lazy=False, back_populates='creating_job_associations') + def __init__(self, name, dataset): self.name = name self.dataset = dataset diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 419327a3f1d..a1f05fdf353 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -359,13 +359,6 @@ model.JobParameter.table = Table( Column("name", String(255)), Column("value", TEXT)) -model.JobToOutputLibraryDatasetAssociation.table = Table( - "job_to_output_library_dataset", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("ldda_id", Integer, ForeignKey("library_dataset_dataset_association.id"), index=True), - Column("name", String(255))) - model.ImplicitlyCreatedDatasetCollectionInput.table = Table( "implicitly_created_dataset_collection_inputs", metadata, Column("id", Integer, primary_key=True), @@ -1334,6 +1327,8 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.L back_populates='library_dataset_dataset_association'), dependent_jobs=relation( model.JobToInputLibraryDatasetAssociation, back_populates='dataset'), + creating_job_associations=relation( + model.JobToOutputLibraryDatasetAssociation, back_populates='dataset'), )) mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, model.LibraryDatasetDatasetInfoAssociation.table, properties=dict( @@ -1350,12 +1345,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -mapper_registry.map_imperatively(model.JobToOutputLibraryDatasetAssociation, model.JobToOutputLibraryDatasetAssociation.table, properties=dict( - dataset=relation(model.LibraryDatasetDatasetAssociation, - lazy=False, - backref="creating_job_associations") -)) - simple_mapping(model.JobStateHistory, job=relation(model.Job, backref="state_history")) @@ -1829,7 +1818,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( post_job_actions=relation(model.PostJobActionAssociation, backref="job", lazy=False), input_library_datasets=relation(model.JobToInputLibraryDatasetAssociation, back_populates="job"), output_library_datasets=relation(model.JobToOutputLibraryDatasetAssociation, - backref="job", lazy=True), + back_populates="job", lazy=True), external_output_metadata=relation(model.JobExternalOutputMetadata, lazy=True, backref='job'), tasks=relation(model.Task, backref='job'), output_datasets=relation(model.JobToOutputDatasetAssociation, back_populates='job'), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 12088c5932c..c202ba63ace 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -789,6 +789,27 @@ def test_JobToOutputDatasetCollectionAssociation( assert stored_obj.dataset_collection_instance.id == history_dataset_collection_association.id +def test_JobToOutputLibraryDatasetAssociation(model, session, library_dataset_dataset_association, job): + cls = model.JobToOutputLibraryDatasetAssociation + assert cls.__tablename__ == 'job_to_output_library_dataset' + + with dbcleanup(session, cls): + name = 'a' + obj = cls(name, library_dataset_dataset_association) + obj.job = job + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.ldda_id == library_dataset_dataset_association.id + assert stored_obj.name == name + # test mapped relationships + assert stored_obj.job.id == job.id + assert stored_obj.dataset.id == library_dataset_dataset_association.id + + def test_Library(model, session, library_folder, library_permission): cls = model.Library assert cls.__tablename__ == 'library' From fdc947c704849ead13f9d44b5479e059bf5fe2aa Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 2 Jul 2021 12:12:15 -0400 Subject: [PATCH 095/225] Map JobStateHistory declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 13 +------------ test/unit/model/test_model.py | 24 ++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 445a4086abc..366ce2dd109 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1700,7 +1700,17 @@ class JobToOutputLibraryDatasetAssociation(Base, RepresentById): self.dataset = dataset -class JobStateHistory(RepresentById): +class JobStateHistory(Base, RepresentById): + __tablename__ = 'job_state_history' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + state = Column('state', String(64), index=True) + info = Column('info', TrimmedString(255)) + job = relationship('Job', back_populates='state_history') + def __init__(self, job): self.job = job self.state = job.state diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index a1f05fdf353..11843c37e2c 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -343,15 +343,6 @@ model.Job.table = Table( Column("params", TrimmedString(255), index=True), Column("handler", TrimmedString(255), index=True)) -model.JobStateHistory.table = Table( - "job_state_history", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("state", String(64), index=True), - Column("info", TrimmedString(255))) - model.JobParameter.table = Table( "job_parameter", metadata, Column("id", Integer, primary_key=True), @@ -1345,9 +1336,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -simple_mapping(model.JobStateHistory, - job=relation(model.Job, backref="state_history")) - simple_mapping(model.JobMetricText, job=relation(model.Job, backref="text_metrics")) @@ -1822,6 +1810,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( external_output_metadata=relation(model.JobExternalOutputMetadata, lazy=True, backref='job'), tasks=relation(model.Task, backref='job'), output_datasets=relation(model.JobToOutputDatasetAssociation, back_populates='job'), + state_history=relation(model.JobStateHistory, back_populates='job'), )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index c202ba63ace..40602a57b64 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -635,6 +635,30 @@ def test_HistoryTagAssociation(model, session, history, tag, user): assert stored_obj.user.id == user.id +def test_JobStateHistory(model, session, job): + cls = model.JobStateHistory + assert cls.__tablename__ == 'job_state_history' + with dbcleanup(session, cls): + state, info = job.state, job.info + obj = cls(job) + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj.create_time = create_time + obj.update_time = update_time + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.job_id == job.id + assert stored_obj.state == state + assert stored_obj.info == info + # test mapped relationships + assert stored_obj.job.id == job.id + + def test_JobToImplicitOutputDatasetCollectionAssociation( model, session, dataset_collection, job): cls = model.JobToImplicitOutputDatasetCollectionAssociation From da27b921c57ab97d46a4e24cb4955b0f0fd51849 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 2 Jul 2021 12:25:53 -0400 Subject: [PATCH 096/225] Map JobMetricText declaratively --- lib/galaxy/model/__init__.py | 11 +++++++++-- lib/galaxy/model/mapping.py | 12 +----------- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 366ce2dd109..5028500a95b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -745,8 +745,15 @@ class BaseJobMetric: self.metric_value = metric_value -class JobMetricText(BaseJobMetric, RepresentById): - pass +class JobMetricText(Base, BaseJobMetric, RepresentById): + __tablename__ = 'job_metric_text' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + plugin = Column('plugin', Unicode(255)) + metric_name = Column('metric_name', Unicode(255)) + metric_value = Column('metric_value', Unicode(JOB_METRIC_MAX_LENGTH)) + job = relationship('Job', back_populates='text_metrics') class JobMetricNumeric(BaseJobMetric, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 11843c37e2c..98f29412acd 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -405,14 +405,6 @@ model.JobImportHistoryArchive.table = Table( Column("history_id", Integer, ForeignKey("history.id"), index=True), Column("archive_dir", TEXT)) -model.JobMetricText.table = Table( - "job_metric_text", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("plugin", Unicode(255)), - Column("metric_name", Unicode(255)), - Column("metric_value", Unicode(model.JOB_METRIC_MAX_LENGTH))) - model.TaskMetricText.table = Table( "task_metric_text", metadata, Column("id", Integer, primary_key=True), @@ -1336,9 +1328,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -simple_mapping(model.JobMetricText, - job=relation(model.Job, backref="text_metrics")) - simple_mapping(model.TaskMetricText, task=relation(model.Task, backref="text_metrics")) @@ -1811,6 +1800,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( tasks=relation(model.Task, backref='job'), output_datasets=relation(model.JobToOutputDatasetAssociation, back_populates='job'), state_history=relation(model.JobStateHistory, back_populates='job'), + text_metrics=relation(model.JobMetricText, back_populates='job'), )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 40602a57b64..4c0c7b82049 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -635,6 +635,26 @@ def test_HistoryTagAssociation(model, session, history, tag, user): assert stored_obj.user.id == user.id +def test_JobMetricText(model, session, job): + cls = model.JobMetricText + assert cls.__tablename__ == 'job_metric_text' + with dbcleanup(session, cls): + plugin, metric_name, metric_value = 'a', 'b', 'c' + obj = cls(plugin, metric_name, metric_value) + obj.job = job + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.plugin == plugin + assert stored_obj.plugin == plugin + assert stored_obj.metric_value == metric_value + # test mapped relationships + assert stored_obj.job.id == job.id + + def test_JobStateHistory(model, session, job): cls = model.JobStateHistory assert cls.__tablename__ == 'job_state_history' From 51625dfbbe80ddd5545a962641437b88448d9bdb Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 2 Jul 2021 13:13:58 -0400 Subject: [PATCH 097/225] Map TaskMetricText declaratively --- lib/galaxy/model/__init__.py | 11 +++++++++-- lib/galaxy/model/mapping.py | 15 +++------------ test/unit/model/test_model.py | 25 +++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 5028500a95b..2162a352ae9 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -760,8 +760,15 @@ class JobMetricNumeric(BaseJobMetric, RepresentById): pass -class TaskMetricText(BaseJobMetric, RepresentById): - pass +class TaskMetricText(Base, BaseJobMetric, RepresentById): + __tablename__ = 'task_metric_text' + + id = Column('id', Integer, primary_key=True) + task_id = Column('task_id', Integer, ForeignKey('task.id'), index=True) + plugin = Column('plugin', Unicode(255)) + metric_name = Column('metric_name', Unicode(255)) + metric_value = Column('metric_value', Unicode(JOB_METRIC_MAX_LENGTH)) + task = relationship('Task', back_populates='text_metrics') class TaskMetricNumeric(BaseJobMetric, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 98f29412acd..d82c8ec9da2 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -405,14 +405,6 @@ model.JobImportHistoryArchive.table = Table( Column("history_id", Integer, ForeignKey("history.id"), index=True), Column("archive_dir", TEXT)) -model.TaskMetricText.table = Table( - "task_metric_text", metadata, - Column("id", Integer, primary_key=True), - Column("task_id", Integer, ForeignKey("task.id"), index=True), - Column("plugin", Unicode(255)), - Column("metric_name", Unicode(255)), - Column("metric_value", Unicode(model.JOB_METRIC_MAX_LENGTH))) - model.JobMetricNumeric.table = Table( "job_metric_numeric", metadata, Column("id", Integer, primary_key=True), @@ -1328,9 +1320,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -simple_mapping(model.TaskMetricText, - task=relation(model.Task, backref="text_metrics")) - simple_mapping(model.JobMetricNumeric, job=relation(model.Job, backref="numeric_metrics")) @@ -1413,7 +1402,9 @@ mapper_registry.map_imperatively(model.PostJobActionAssociation, model.PostJobAc post_job_action=relation(model.PostJobAction) )) -mapper_registry.map_imperatively(model.Task, model.Task.table) +mapper_registry.map_imperatively(model.Task, model.Task.table, properties=dict( + text_metrics=relation(model.TaskMetricText, back_populates='task'), +)) mapper_registry.map_imperatively(model.DeferredJob, model.DeferredJob.table, properties={}) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 4c0c7b82049..40c4f8e022f 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1574,6 +1574,25 @@ def test_Tag( assert stored_obj.tagged_tools == [tool_tag_association] +def test_TaskMetricText(model, session, task): + cls = model.TaskMetricText + assert cls.__tablename__ == 'task_metric_text' + with dbcleanup(session, cls): + plugin, metric_name, metric_value = 'a', 'b', 'c' + obj = cls(plugin, metric_name, metric_value) + obj.task = task + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.task_id == task.id + assert stored_obj.plugin == plugin + assert stored_obj.metric_value == metric_value + # test mapped relationships + assert stored_obj.task.id == task.id + + def test_ToolTagAssociation(model, session, tag, user): cls = model.ToolTagAssociation assert cls.__tablename__ == 'tool_tag_association' @@ -2165,6 +2184,12 @@ def tag(model, session): yield from dbcleanup_wrapper(session, t) +@pytest.fixture +def task(model, session, job): + t = model.Task(job, 'a', 'b') + yield from dbcleanup_wrapper(session, t) + + @pytest.fixture def tool_tag_association(model, session): tta = model.ToolTagAssociation() From ebd74d1ff360484c1785730d463b77f2941f8362 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 2 Jul 2021 13:23:41 -0400 Subject: [PATCH 098/225] Map TaskMetricNumeric declaratively --- lib/galaxy/model/__init__.py | 12 ++++++++++-- lib/galaxy/model/mapping.py | 13 +------------ test/unit/model/test_model.py | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 2162a352ae9..517e74e340e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -45,6 +45,7 @@ from sqlalchemy import ( Integer, join, not_, + Numeric, or_, PrimaryKeyConstraint, select, @@ -771,8 +772,15 @@ class TaskMetricText(Base, BaseJobMetric, RepresentById): task = relationship('Task', back_populates='text_metrics') -class TaskMetricNumeric(BaseJobMetric, RepresentById): - pass +class TaskMetricNumeric(Base, BaseJobMetric, RepresentById): + __tablename__ = 'task_metric_numeric' + + id = Column('id', Integer, primary_key=True) + task_id = Column('task_id', Integer, ForeignKey('task.id'), index=True) + plugin = Column('plugin', Unicode(255)) + metric_name = Column('metric_name', Unicode(255)) + metric_value = Column('metric_value', Numeric(JOB_METRIC_PRECISION, JOB_METRIC_SCALE)) + task = relationship('Task', back_populates='numeric_metrics') class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index d82c8ec9da2..859828bd280 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -413,15 +413,6 @@ model.JobMetricNumeric.table = Table( Column("metric_name", Unicode(255)), Column("metric_value", Numeric(model.JOB_METRIC_PRECISION, model.JOB_METRIC_SCALE))) -model.TaskMetricNumeric.table = Table( - "task_metric_numeric", metadata, - Column("id", Integer, primary_key=True), - Column("task_id", Integer, ForeignKey("task.id"), index=True), - Column("plugin", Unicode(255)), - Column("metric_name", Unicode(255)), - Column("metric_value", Numeric(model.JOB_METRIC_PRECISION, model.JOB_METRIC_SCALE))) - - model.GenomeIndexToolData.table = Table( "genome_index_tool_data", metadata, Column("id", Integer, primary_key=True), @@ -1323,9 +1314,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod simple_mapping(model.JobMetricNumeric, job=relation(model.Job, backref="numeric_metrics")) -simple_mapping(model.TaskMetricNumeric, - task=relation(model.Task, backref="numeric_metrics")) - simple_mapping(model.ImplicitlyCreatedDatasetCollectionInput, input_dataset_collection=relation(model.HistoryDatasetCollectionAssociation, primaryjoin=(model.HistoryDatasetCollectionAssociation.table.c.id @@ -1404,6 +1392,7 @@ mapper_registry.map_imperatively(model.PostJobActionAssociation, model.PostJobAc mapper_registry.map_imperatively(model.Task, model.Task.table, properties=dict( text_metrics=relation(model.TaskMetricText, back_populates='task'), + numeric_metrics=relation(model.TaskMetricNumeric, back_populates='task'), )) mapper_registry.map_imperatively(model.DeferredJob, model.DeferredJob.table, properties={}) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 40c4f8e022f..0c16ecc390d 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1574,6 +1574,25 @@ def test_Tag( assert stored_obj.tagged_tools == [tool_tag_association] +def test_TaskMetricNumeric(model, session, task): + cls = model.TaskMetricNumeric + assert cls.__tablename__ == 'task_metric_numeric' + with dbcleanup(session, cls): + plugin, metric_name, metric_value = 'a', 'b', 9 + obj = cls(plugin, metric_name, metric_value) + obj.task = task + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.task_id == task.id + assert stored_obj.plugin == plugin + assert stored_obj.metric_value == metric_value + # test mapped relationships + assert stored_obj.task.id == task.id + + def test_TaskMetricText(model, session, task): cls = model.TaskMetricText assert cls.__tablename__ == 'task_metric_text' From 071a70a6988069332b5adf9425b7886d6fabcfa2 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 2 Jul 2021 13:29:36 -0400 Subject: [PATCH 099/225] Map JobMetricNumeric declaratively --- lib/galaxy/model/__init__.py | 11 +++++++++-- lib/galaxy/model/mapping.py | 12 +----------- test/unit/model/test_model.py | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 517e74e340e..98f9c6751af 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -757,8 +757,15 @@ class JobMetricText(Base, BaseJobMetric, RepresentById): job = relationship('Job', back_populates='text_metrics') -class JobMetricNumeric(BaseJobMetric, RepresentById): - pass +class JobMetricNumeric(Base, BaseJobMetric, RepresentById): + __tablename__ = 'job_metric_numeric' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + plugin = Column('plugin', Unicode(255)) + metric_name = Column('metric_name', Unicode(255)) + metric_value = Column('metric_value', Numeric(JOB_METRIC_PRECISION, JOB_METRIC_SCALE)) + job = relationship('Job', back_populates='numeric_metrics') class TaskMetricText(Base, BaseJobMetric, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 859828bd280..346ab461074 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -405,14 +405,6 @@ model.JobImportHistoryArchive.table = Table( Column("history_id", Integer, ForeignKey("history.id"), index=True), Column("archive_dir", TEXT)) -model.JobMetricNumeric.table = Table( - "job_metric_numeric", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("plugin", Unicode(255)), - Column("metric_name", Unicode(255)), - Column("metric_value", Numeric(model.JOB_METRIC_PRECISION, model.JOB_METRIC_SCALE))) - model.GenomeIndexToolData.table = Table( "genome_index_tool_data", metadata, Column("id", Integer, primary_key=True), @@ -1311,9 +1303,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -simple_mapping(model.JobMetricNumeric, - job=relation(model.Job, backref="numeric_metrics")) - simple_mapping(model.ImplicitlyCreatedDatasetCollectionInput, input_dataset_collection=relation(model.HistoryDatasetCollectionAssociation, primaryjoin=(model.HistoryDatasetCollectionAssociation.table.c.id @@ -1781,6 +1770,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( output_datasets=relation(model.JobToOutputDatasetAssociation, back_populates='job'), state_history=relation(model.JobStateHistory, back_populates='job'), text_metrics=relation(model.JobMetricText, back_populates='job'), + numeric_metrics=relation(model.JobMetricNumeric, back_populates='job'), )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 0c16ecc390d..b1fea8cac94 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -635,6 +635,26 @@ def test_HistoryTagAssociation(model, session, history, tag, user): assert stored_obj.user.id == user.id +def test_JobMetricNumeric(model, session, job): + cls = model.JobMetricNumeric + assert cls.__tablename__ == 'job_metric_numeric' + with dbcleanup(session, cls): + plugin, metric_name, metric_value = 'a', 'b', 9 + obj = cls(plugin, metric_name, metric_value) + obj.job = job + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.plugin == plugin + assert stored_obj.plugin == plugin + assert stored_obj.metric_value == metric_value + # test mapped relationships + assert stored_obj.job.id == job.id + + def test_JobMetricText(model, session, job): cls = model.JobMetricText assert cls.__tablename__ == 'job_metric_text' From b5bcd6987c1a4768373610a86987b159bb588633 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 2 Jul 2021 22:42:09 -0400 Subject: [PATCH 100/225] Map JobParameter declaratively --- lib/galaxy/model/__init__.py | 9 ++++++++- lib/galaxy/model/mapping.py | 9 --------- test/unit/model/test_model.py | 19 +++++++++++++++++-- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 98f9c6751af..6447772f17d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1582,7 +1582,14 @@ class Task(JobLike, RepresentById): self.prepare_input_files_cmd = prepare_input_files_cmd -class JobParameter(RepresentById): +class JobParameter(Base, RepresentById): + __tablename__ = 'job_parameter' + + id = Column('id', Integer, primary_key=True) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) + name = Column('name', String(255)) + value = Column('value', TEXT) + def __init__(self, name, value): self.name = name self.value = value diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 346ab461074..0540fe068a9 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -343,13 +343,6 @@ model.Job.table = Table( Column("params", TrimmedString(255), index=True), Column("handler", TrimmedString(255), index=True)) -model.JobParameter.table = Table( - "job_parameter", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("name", String(255)), - Column("value", TEXT)) - model.ImplicitlyCreatedDatasetCollectionInput.table = Table( "implicitly_created_dataset_collection_inputs", metadata, Column("id", Integer, primary_key=True), @@ -1336,8 +1329,6 @@ simple_mapping( ), ) -mapper_registry.map_imperatively(model.JobParameter, model.JobParameter.table) - mapper_registry.map_imperatively(model.JobExternalOutputMetadata, model.JobExternalOutputMetadata.table, properties=dict( history_dataset_association=relation(model.HistoryDatasetAssociation, lazy=False), library_dataset_dataset_association=relation(model.LibraryDatasetDatasetAssociation, lazy=False) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index b1fea8cac94..5ab8061813a 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -649,7 +649,6 @@ def test_JobMetricNumeric(model, session, job): assert stored_obj.id == obj_id assert stored_obj.job_id == job.id assert stored_obj.plugin == plugin - assert stored_obj.plugin == plugin assert stored_obj.metric_value == metric_value # test mapped relationships assert stored_obj.job.id == job.id @@ -669,12 +668,28 @@ def test_JobMetricText(model, session, job): assert stored_obj.id == obj_id assert stored_obj.job_id == job.id assert stored_obj.plugin == plugin - assert stored_obj.plugin == plugin assert stored_obj.metric_value == metric_value # test mapped relationships assert stored_obj.job.id == job.id +def test_JobParameter(model, session, job): + cls = model.JobParameter + assert cls.__tablename__ == 'job_parameter' + with dbcleanup(session, cls): + name, value = 'a', 'b' + obj = cls(name, value) + obj.job_id = job.id + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.name == name + assert stored_obj.value == value + + def test_JobStateHistory(model, session, job): cls = model.JobStateHistory assert cls.__tablename__ == 'job_state_history' From 1f1d82bfd28732e41a3656bffa158fad33b88490 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 3 Jul 2021 11:57:12 -0400 Subject: [PATCH 101/225] Map WorkflowRequestInputParameter declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 14 +------------- test/unit/model/test_model.py | 27 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6447772f17d..70763903597 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6280,9 +6280,19 @@ class WorkflowInvocationStep(Dictifiable, RepresentById): return rval -class WorkflowRequestInputParameter(Dictifiable, RepresentById): +class WorkflowRequestInputParameter(Base, Dictifiable, RepresentById): """ Workflow-related parameters not tied to steps or inputs. """ + __tablename__ = 'workflow_request_input_parameters' + + id = Column('id', Integer, primary_key=True) + workflow_invocation_id = Column('workflow_invocation_id', Integer, + ForeignKey('workflow_invocation.id', onupdate='CASCADE', ondelete='CASCADE')) + name = Column('name', Unicode(255)) + value = Column('value', TEXT) + type = Column('type', Unicode(255)) + workflow_invocation = relationship('WorkflowInvocation', back_populates='input_parameters') + dict_collection_visible_keys = ['id', 'name', 'value', 'type'] class types(str, Enum): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 0540fe068a9..76a0fc3486e 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -661,15 +661,6 @@ model.WorkflowRequestStepState.table = Table( Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")), Column("value", MutableJSONType)) -model.WorkflowRequestInputParameter.table = Table( - "workflow_request_input_parameters", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_invocation_id", Integer, - ForeignKey("workflow_invocation.id", onupdate="CASCADE", ondelete="CASCADE")), - Column("name", Unicode(255)), - Column("value", TEXT), - Column("type", Unicode(255))) - model.WorkflowRequestInputStepParameter.table = Table( "workflow_request_input_step_parameter", metadata, Column("id", Integer, primary_key=True), @@ -1583,7 +1574,7 @@ mapper_registry.map_imperatively(model.StoredWorkflowMenuEntry, model.StoredWork mapper_registry.map_imperatively(model.WorkflowInvocation, model.WorkflowInvocation.table, properties=dict( history=relation(model.History, backref=backref('workflow_invocations', uselist=True)), - input_parameters=relation(model.WorkflowRequestInputParameter, backref='workflow_invocation'), + input_parameters=relation(model.WorkflowRequestInputParameter, back_populates='workflow_invocation'), step_states=relation(model.WorkflowRequestStepState, backref='workflow_invocation'), input_step_parameters=relation(model.WorkflowRequestInputStepParameter, backref='workflow_invocation'), @@ -1622,9 +1613,6 @@ simple_mapping(model.WorkflowInvocationStep, ), ) - -simple_mapping(model.WorkflowRequestInputParameter) - simple_mapping(model.WorkflowRequestStepState, workflow_step=relation(model.WorkflowStep)) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 5ab8061813a..7b104494637 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1961,6 +1961,26 @@ def test_VisualizationTagAssociation(model, session, visualization, tag, user): assert stored_obj.user.id == user.id +def test_WorkflowRequestInputParameter(model, session, workflow_invocation): + cls = model.WorkflowRequestInputParameter + assert cls.__tablename__ == 'workflow_request_input_parameters' + with dbcleanup(session, cls): + name, value, type = 'a', 'b', 'c' + obj = cls(name, value, type) + obj.workflow_invocation = workflow_invocation + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.name == name + assert stored_obj.value == value + assert stored_obj.type == type + # test mapped relationships + assert stored_obj.workflow_invocation.id == workflow_invocation.id + + @pytest.fixture(scope='module') def model(): db_uri = 'sqlite:///:memory:' @@ -2287,6 +2307,13 @@ def workflow(model, session): yield from dbcleanup_wrapper(session, w) +@pytest.fixture +def workflow_invocation(model, session, workflow): + wi = model.WorkflowInvocation() + wi.workflow = workflow + yield from dbcleanup_wrapper(session, wi) + + @pytest.fixture def workflow_step(model, session, workflow): s = model.WorkflowStep() From a39da9e7a7dad449401dc85148b8b6f3e5b94b4b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 6 Jul 2021 15:18:19 -0400 Subject: [PATCH 102/225] Map Task declaratively --- lib/galaxy/model/__init__.py | 29 +++++++++++- lib/galaxy/model/mapping.py | 31 +------------ test/unit/model/test_model.py | 84 +++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 31 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 70763903597..4c17cedbf08 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1431,10 +1431,37 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): return False -class Task(JobLike, RepresentById): +class Task(Base, JobLike, RepresentById): """ A task represents a single component of a job. """ + __tablename__ = 'task' + + id = Column('id', Integer, primary_key=True) + create_time = Column('create_time', DateTime, default=now) + execution_time = Column('execution_time', DateTime) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + state = Column('state', String(64), index=True) + command_line = Column('command_line', TEXT) + param_filename = Column('param_filename', String(1024)) + runner_name = Column('runner_name', String(255)) + job_stdout = Column('job_stdout', TEXT) # job_stdout makes sense here because it is short for job script standard out + job_stderr = Column('job_stderr', TEXT) + tool_stdout = Column('tool_stdout', TEXT) + tool_stderr = Column('tool_stderr', TEXT) + exit_code = Column('exit_code', Integer, nullable=True) + job_messages = Column('job_messages', MutableJSONType, nullable=True) + info = Column('info', TrimmedString(255)) + traceback = Column('traceback', TEXT) + job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True, nullable=False) + working_directory = Column('working_directory', String(1024)) + task_runner_name = Column('task_runner_name', String(255)) + task_runner_external_id = Column('task_runner_external_id', String(255)) + prepare_input_files_cmd = Column('prepare_input_files_cmd', TEXT) + job = relationship('Job', back_populates='tasks') + text_metrics = relationship('TaskMetricText', back_populates='task') + numeric_metrics = relationship('TaskMetricNumeric', back_populates='task') + _numeric_metric = TaskMetricNumeric _text_metric = TaskMetricText diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 76a0fc3486e..4fa191db7e3 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -439,30 +439,6 @@ model.JobContainerAssociation.table = Table( Column("created_time", DateTime, default=now), Column("modified_time", DateTime, default=now, onupdate=now)) -model.Task.table = Table( - "task", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("execution_time", DateTime), - Column("update_time", DateTime, default=now, onupdate=now), - Column("state", String(64), index=True), - Column("command_line", TEXT), - Column("param_filename", String(1024)), - Column("runner_name", String(255)), - Column("job_stdout", TEXT), # job_stdout makes sense here because it is short for job script standard out. - Column("job_stderr", TEXT), - Column("tool_stdout", TEXT), - Column("tool_stderr", TEXT), - Column("exit_code", Integer, nullable=True), - Column("job_messages", MutableJSONType, nullable=True), - Column("info", TrimmedString(255)), - Column("traceback", TEXT), - Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=False), - Column("working_directory", String(1024)), - Column("task_runner_name", String(255)), - Column("task_runner_external_id", String(255)), - Column("prepare_input_files_cmd", TEXT)) - model.PostJobAction.table = Table( "post_job_action", metadata, Column("id", Integer, primary_key=True), @@ -1361,11 +1337,6 @@ mapper_registry.map_imperatively(model.PostJobActionAssociation, model.PostJobAc post_job_action=relation(model.PostJobAction) )) -mapper_registry.map_imperatively(model.Task, model.Task.table, properties=dict( - text_metrics=relation(model.TaskMetricText, back_populates='task'), - numeric_metrics=relation(model.TaskMetricNumeric, back_populates='task'), -)) - mapper_registry.map_imperatively(model.DeferredJob, model.DeferredJob.table, properties={}) mapper_registry.map_imperatively(model.TransferJob, model.TransferJob.table, properties={}) @@ -1745,7 +1716,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( output_library_datasets=relation(model.JobToOutputLibraryDatasetAssociation, back_populates="job", lazy=True), external_output_metadata=relation(model.JobExternalOutputMetadata, lazy=True, backref='job'), - tasks=relation(model.Task, backref='job'), + tasks=relation(model.Task, back_populates='job'), output_datasets=relation(model.JobToOutputDatasetAssociation, back_populates='job'), state_history=relation(model.JobStateHistory, back_populates='job'), text_metrics=relation(model.JobMetricText, back_populates='job'), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 7b104494637..5e466b607af 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1609,6 +1609,78 @@ def test_Tag( assert stored_obj.tagged_tools == [tool_tag_association] +def test_Task(model, session, job, task_metric_numeric, task_metric_text): + cls = model.Task + assert cls.__tablename__ == 'task' + with dbcleanup(session, cls): + create_time = datetime.now() + execution_time = create_time + timedelta(hours=1) + update_time = execution_time + timedelta(hours=1) + state = model.Task.states.WAITING + command_line = 'a' + param_filename = 'b' + runner_name = 'c' + job_stdout = 'd' + job_stderr = 'e' + tool_stdout = 'f' + tool_stderr = 'g' + exit_code = 9 + job_messages = 'h' + info = 'i' + traceback = 'j' + working_directory = 'k' + task_runner_name = 'l' + task_runner_external_id = 'm' + prepare_input_files_cmd = 'n' + + obj = cls(job, working_directory, prepare_input_files_cmd) + obj.create_time = create_time + obj.execution_time = execution_time + obj.update_time = update_time + obj.state = state + obj.command_line = command_line + obj.param_filename = param_filename + obj.runner_name = runner_name + obj.job_stdout = job_stdout + obj.job_stderr = job_stderr + obj.tool_stdout = tool_stdout + obj.tool_stderr = tool_stderr + obj.exit_code = exit_code + obj.job_messages = job_messages + obj.info = info + obj.traceback = traceback + obj.task_runner_name = task_runner_name + obj.task_runner_external_id = task_runner_external_id + obj.numeric_metrics.append(task_metric_numeric) + obj.text_metrics.append(task_metric_text) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.execution_time == execution_time + assert stored_obj.update_time == update_time + assert stored_obj.state == state + assert stored_obj.command_line == command_line + assert stored_obj.param_filename == param_filename + assert stored_obj.runner_name == runner_name + assert stored_obj.job_stdout == job_stdout + assert stored_obj.job_stderr == job_stderr + assert stored_obj.tool_stdout == tool_stdout + assert stored_obj.tool_stderr == tool_stderr + assert stored_obj.exit_code == exit_code + assert stored_obj.job_messages == job_messages + assert stored_obj.info == info + assert stored_obj.traceback == traceback + assert stored_obj.task_runner_name == task_runner_name + assert stored_obj.task_runner_external_id == task_runner_external_id + # test mapped relationships + assert stored_obj.job.id == job.id + assert stored_obj.numeric_metrics == [task_metric_numeric] + assert stored_obj.text_metrics == [task_metric_text] + + def test_TaskMetricNumeric(model, session, task): cls = model.TaskMetricNumeric assert cls.__tablename__ == 'task_metric_numeric' @@ -2264,6 +2336,18 @@ def task(model, session, job): yield from dbcleanup_wrapper(session, t) +@pytest.fixture +def task_metric_numeric(model, session): + tmn = model. TaskMetricNumeric('a', 'b', 9) + yield from dbcleanup_wrapper(session, tmn) + + +@pytest.fixture +def task_metric_text(model, session): + tmt = model. TaskMetricText('a', 'b', 'c') + yield from dbcleanup_wrapper(session, tmt) + + @pytest.fixture def tool_tag_association(model, session): tta = model.ToolTagAssociation() From 34d83845a469e7228637841959eabdb9e1ad85c9 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 6 Jul 2021 16:17:32 -0400 Subject: [PATCH 103/225] Map HistoryUserShareAssociation declaratively --- lib/galaxy/model/__init__.py | 10 +++++++++- lib/galaxy/model/mapping.py | 16 ++++------------ test/unit/model/test_model.py | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4c17cedbf08..21fed5015b4 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2529,7 +2529,15 @@ class UserShareAssociation(RepresentById): user: Optional[User] -class HistoryUserShareAssociation(UserShareAssociation): +class HistoryUserShareAssociation(Base, UserShareAssociation): + __tablename__ = 'history_user_share_association' + + id = Column('id', Integer, primary_key=True) + history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) + user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + user = relationship('User', back_populates='histories_shared_by_others') + history = relationship('History', back_populates='users_shared_with') + def __init__(self): self.history = None self.user = None diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 4fa191db7e3..023fc1d8045 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -91,12 +91,6 @@ model.History.table = Table( Index('ix_history_slug', 'slug', mysql_length=200), ) -model.HistoryUserShareAssociation.table = Table( - "history_user_share_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True)) - model.HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata, Column("id", Integer, primary_key=True), @@ -1045,13 +1039,14 @@ mapper_registry.map_imperatively(model.History, model.History.table, properties= deferred=True ), users_shared_with_count=column_property( - select(func.count(model.HistoryUserShareAssociation.table.c.id)).where(model.History.table.c.id == model.HistoryUserShareAssociation.table.c.history_id).scalar_subquery(), + select(func.count(model.HistoryUserShareAssociation.id)).where(model.History.table.c.id == model.HistoryUserShareAssociation.history_id).scalar_subquery(), deferred=True ), update_time=column_property( select(func.max(model.HistoryAudit.update_time)).where(model.HistoryAudit.history_id == model.History.table.c.id).scalar_subquery(), ), default_permissions=relation(model.DefaultHistoryPermissions, back_populates='history'), + users_shared_with=relation(model.HistoryUserShareAssociation, back_populates='history'), )) # Set up proxy so that @@ -1059,11 +1054,6 @@ mapper_registry.map_imperatively(model.History, model.History.table, properties= # returns a list of users that history is shared with. model.History.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore -mapper_registry.map_imperatively(model.HistoryUserShareAssociation, model.HistoryUserShareAssociation.table, properties=dict( - user=relation(model.User, backref='histories_shared_by_others'), - history=relation(model.History, backref='users_shared_with') -)) - mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( addresses=relation(model.UserAddress, back_populates='user', @@ -1107,6 +1097,8 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( order_by=desc(model.APIKeys.create_time)), pages=relation(model.Page, back_populates='user'), reset_tokens=relation(model.PasswordResetToken, back_populates='user'), + histories_shared_by_others=relation( + model.HistoryUserShareAssociation, back_populates='user'), )) # Set up proxy so that this syntax is possible: diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 5e466b607af..18f6a562ecc 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -413,6 +413,24 @@ def test_GroupRoleAssociation(model, session, group, role): assert stored_obj.role.id == role.id +def test_HistoryUserShareAssociation(model, session, history, user): + cls = model.HistoryUserShareAssociation + assert cls.__tablename__ == 'history_user_share_association' + with dbcleanup(session, cls): + obj = cls() + obj.history = history + obj.user = user + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id + # test mapped relationships + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id + + def test_HistoryAudit(model, session, history): cls = model.HistoryAudit assert cls.__tablename__ == 'history_audit' From 77400c59718f9da458dd06ed685bfcf8e508f66a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 7 Jul 2021 14:50:12 -0400 Subject: [PATCH 104/225] Map LibraryFolder declaratively --- lib/galaxy/model/__init__.py | 53 +++++++++++++++++++++++++++- lib/galaxy/model/mapping.py | 53 +--------------------------- test/unit/model/test_model.py | 65 +++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 53 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 21fed5015b4..41c20fe9948 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -34,6 +34,7 @@ from social_core.storage import AssociationMixin, CodeMixin, NonceMixin, Partial from sqlalchemy import ( alias, and_, + asc, BigInteger, Boolean, Column, @@ -4169,7 +4170,57 @@ class Library(Base, Dictifiable, HasName, RepresentById): return roles -class LibraryFolder(Dictifiable, HasName, RepresentById): +class LibraryFolder(Base, Dictifiable, HasName, RepresentById): + __tablename__ = 'library_folder' + __table_args__ = ( + Index('ix_library_folder_name', 'name', mysql_length=200), + ) + + id = Column('id', Integer, primary_key=True) + parent_id = Column('parent_id', Integer, ForeignKey('library_folder.id'), nullable=True, index=True) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + name = Column('name', TEXT) + description = Column('description', TEXT) + order_id = Column('order_id', Integer) # not currently being used, but for possible future use + item_count = Column('item_count', Integer) + deleted = Column('deleted', Boolean, index=True, default=False) + purged = Column('purged', Boolean, index=True, default=False) + genome_build = Column('genome_build', TrimmedString(40)) + + folders = relationship('LibraryFolder', + primaryjoin=('LibraryFolder.id == LibraryFolder.parent_id'), + order_by=asc(name), + back_populates='parent') + parent = relationship('LibraryFolder', back_populates='folders', remote_side=[id]) + + active_folders = relationship('LibraryFolder', + primaryjoin=( + 'and_(LibraryFolder.parent_id == LibraryFolder.id, not_(LibraryFolder.deleted))'), + order_by=asc(name), + # """sqlalchemy.exc.ArgumentError: Error creating eager relationship 'active_folders' + # on parent class '' to child class '': + # Cant use eager loading on a self referential relationship.""" + # TODO: This is no longer the case. Fix this: https://docs.sqlalchemy.org/en/14/orm/self_referential.html#configuring-self-referential-eager-loading + lazy=True, + viewonly=True) + + datasets = relationship('LibraryDataset', + primaryjoin=('LibraryDataset.folder_id == LibraryFolder.id'), + # order_by=asc(text('LibraryDataset._name')), # TODO: uncomment after mapping LibraryDataset + lazy=True, + viewonly=True) + + active_datasets = relationship('LibraryDataset', + primaryjoin=( + 'and_(LibraryDataset.folder_id == LibraryFolder.id, not_(LibraryDataset.deleted))'), + # order_by=asc(text('LibraryDataset._name')), # TODO: uncomment after mapping LibraryDataset + lazy=True, + viewonly=True) + + library_root = relationship('Library', back_populates='root_folder') + actions = relationship('LibraryFolderPermissions', back_populates='folder') + dict_element_visible_keys = ['id', 'parent_id', 'name', 'description', 'item_count', 'genome_build', 'update_time', 'deleted'] def __init__(self, name=None, description=None, item_count=0, order_id=None, genome_build=None): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 023fc1d8045..8af78968233 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -258,22 +258,6 @@ model.ExtendedMetadataIndex.table = Table( Column("path", String(255)), Column("value", TEXT)) -model.LibraryFolder.table = Table( - "library_folder", metadata, - Column("id", Integer, primary_key=True), - Column("parent_id", Integer, ForeignKey("library_folder.id"), nullable=True, index=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("name", TEXT), - Column("description", TEXT), - Column("order_id", Integer), # not currently being used, but for possible future use - Column("item_count", Integer), - Column("deleted", Boolean, index=True, default=False), - Column("purged", Boolean, index=True, default=False), - Column("genome_build", TrimmedString(40)), - Index('ix_library_folder_name', 'name', mysql_length=200), -) - model.LibraryInfoAssociation.table = Table( "library_info_association", metadata, Column("id", Integer, primary_key=True), @@ -1138,45 +1122,10 @@ mapper_registry.map_imperatively(model.LibraryInfoAssociation, model.LibraryInfo primaryjoin=(model.LibraryInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -mapper_registry.map_imperatively(model.LibraryFolder, model.LibraryFolder.table, properties=dict( - folders=relation(model.LibraryFolder, - primaryjoin=(model.LibraryFolder.table.c.parent_id == model.LibraryFolder.table.c.id), - order_by=asc(model.LibraryFolder.table.c.name), - backref=backref("parent", - primaryjoin=(model.LibraryFolder.table.c.parent_id == model.LibraryFolder.table.c.id), - remote_side=[model.LibraryFolder.table.c.id])), - active_folders=relation(model.LibraryFolder, - primaryjoin=( - (model.LibraryFolder.table.c.parent_id == model.LibraryFolder.table.c.id) - & (not_(model.LibraryFolder.table.c.deleted)) - ), - order_by=asc(model.LibraryFolder.table.c.name), - # """sqlalchemy.exc.ArgumentError: Error creating eager relationship 'active_folders' - # on parent class '' to child class '': - # Cant use eager loading on a self referential relationship.""" - lazy=True, - viewonly=True), - datasets=relation(model.LibraryDataset, - primaryjoin=(model.LibraryDataset.table.c.folder_id == model.LibraryFolder.table.c.id), - order_by=asc(model.LibraryDataset.table.c._name), - lazy=True, - viewonly=True), - active_datasets=relation(model.LibraryDataset, - primaryjoin=( - (model.LibraryDataset.table.c.folder_id == model.LibraryFolder.table.c.id) - & (not_(model.LibraryDataset.table.c.deleted)) - ), - order_by=asc(model.LibraryDataset.table.c._name), - lazy=True, - viewonly=True), - library_root=relation(model.Library, back_populates='root_folder'), - actions=relation(model.LibraryFolderPermissions, back_populates='folder'), -)) - mapper_registry.map_imperatively(model.LibraryFolderInfoAssociation, model.LibraryFolderInfoAssociation.table, properties=dict( folder=relation(model.LibraryFolder, primaryjoin=( - (model.LibraryFolderInfoAssociation.table.c.library_folder_id == model.LibraryFolder.table.c.id) + (model.LibraryFolderInfoAssociation.table.c.library_folder_id == model.LibraryFolder.id) & (not_(model.LibraryFolderInfoAssociation.table.c.deleted)) ), backref="info_association"), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 18f6a562ecc..0d2a34d1f5f 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1083,6 +1083,71 @@ def test_LibraryDatasetPermissions(model, session, library_dataset, role): assert stored_obj.role.id == role.id +def test_LibraryFolder( + model, session, library_folder, library_dataset, library, library_folder_permission): + cls = model.LibraryFolder + assert cls.__tablename__ == 'library_folder' + assert has_index(cls.__table__, ('name',)) + with dbcleanup(session, cls): + name = 'a' + description = 'b' + order_id = 9 + item_count = 1 + deleted = False + purged = False + genome_build = 'c' + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + folder1 = model.LibraryFolder() + + obj = cls() + obj.create_time = create_time + obj.update_time = update_time + obj.parent = library_folder + obj.create_time = create_time + obj.update_time = update_time + obj.name = name + obj.description = description + obj.order_id = order_id + obj.item_count = item_count + obj.deleted = deleted + obj.purged = purged + obj.genome_build = genome_build + obj.folders.append(folder1) + + # No back reference, so dataset does not update folder; so we have to flush to the db + # TODO: ..But why is there no back reference? + library_dataset.folder = obj + persist(session, library_dataset) + + obj.library_root.append(library) + obj.actions.append(library_folder_permission) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls, obj_id) + # test mapped columns + assert stored_obj.id == obj_id + assert stored_obj.parent_id == library_folder.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.description == description + assert stored_obj.order_id == order_id + assert stored_obj.item_count == item_count + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + assert stored_obj.genome_build == genome_build + # test mapped relationships + assert stored_obj.parent.id == library_folder.id + assert stored_obj.folders == [folder1] + assert stored_obj.active_folders == [folder1] + assert stored_obj.library_root == [library] + assert stored_obj.actions == [library_folder_permission] + # Use identity equality instread of object equality. + assert stored_obj.datasets[0].id == library_dataset.id + assert stored_obj.active_datasets[0].id == library_dataset.id + + def test_LibraryFolderPermissions(model, session, library_folder, role): cls = model.LibraryFolderPermissions assert cls.__tablename__ == 'library_folder_permissions' From 0fec0d126a8ca717970bbcea06fb19570cbcf414 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 7 Jul 2021 15:24:12 -0400 Subject: [PATCH 105/225] Refactor LibraryFolder test Is this cleaner vs. one big test? --- test/unit/model/test_model.py | 130 +++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 59 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 0d2a34d1f5f..59dd0df39f5 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1083,69 +1083,81 @@ def test_LibraryDatasetPermissions(model, session, library_dataset, role): assert stored_obj.role.id == role.id -def test_LibraryFolder( - model, session, library_folder, library_dataset, library, library_folder_permission): - cls = model.LibraryFolder - assert cls.__tablename__ == 'library_folder' - assert has_index(cls.__table__, ('name',)) - with dbcleanup(session, cls): - name = 'a' - description = 'b' - order_id = 9 - item_count = 1 - deleted = False - purged = False - genome_build = 'c' - create_time = datetime.now() - update_time = create_time + timedelta(hours=1) - folder1 = model.LibraryFolder() +class TestLibraryFolder: - obj = cls() - obj.create_time = create_time - obj.update_time = update_time - obj.parent = library_folder - obj.create_time = create_time - obj.update_time = update_time - obj.name = name - obj.description = description - obj.order_id = order_id - obj.item_count = item_count - obj.deleted = deleted - obj.purged = purged - obj.genome_build = genome_build - obj.folders.append(folder1) + @pytest.fixture + def cls_(self, model): + return model.LibraryFolder - # No back reference, so dataset does not update folder; so we have to flush to the db - # TODO: ..But why is there no back reference? - library_dataset.folder = obj - persist(session, library_dataset) + def test_table(self, cls_, model): + assert cls_.__tablename__ == 'library_folder' + assert has_index(cls_.__table__, ('name',)) - obj.library_root.append(library) - obj.actions.append(library_folder_permission) - obj_id = persist(session, obj) + def test_columns(self, cls_, model, session, library_folder): + with dbcleanup(session, cls_): + parent = library_folder + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + name = 'a' + description = 'b' + order_id = 9 + item_count = 1 + deleted = False + purged = False + genome_build = 'c' - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.parent_id == library_folder.id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.name == name - assert stored_obj.description == description - assert stored_obj.order_id == order_id - assert stored_obj.item_count == item_count - assert stored_obj.deleted == deleted - assert stored_obj.purged == purged - assert stored_obj.genome_build == genome_build - # test mapped relationships - assert stored_obj.parent.id == library_folder.id - assert stored_obj.folders == [folder1] - assert stored_obj.active_folders == [folder1] - assert stored_obj.library_root == [library] - assert stored_obj.actions == [library_folder_permission] - # Use identity equality instread of object equality. - assert stored_obj.datasets[0].id == library_dataset.id - assert stored_obj.active_datasets[0].id == library_dataset.id + obj = cls_() + obj.parent = parent + obj.create_time = create_time + obj.update_time = update_time + obj.name = name + obj.description = description + obj.order_id = order_id + obj.item_count = item_count + obj.deleted = deleted + obj.purged = purged + obj.genome_build = genome_build + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.parent_id == parent.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.description == description + assert stored_obj.order_id == order_id + assert stored_obj.item_count == item_count + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + assert stored_obj.genome_build == genome_build + + def test_relationships( + self, cls_, model, session, library_folder, library_dataset, library, library_folder_permission): + with dbcleanup(session, cls_): + folder1 = model.LibraryFolder() + + obj = cls_() + # no back reference, so dataset does not update folder; so we have to flush to the db + # todo: ..but why is there no back reference? + library_dataset.folder = obj + persist(session, library_dataset) + + obj.parent = library_folder + obj.folders.append(folder1) + obj.library_root.append(library) + obj.actions.append(library_folder_permission) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.parent.id == library_folder.id + assert stored_obj.folders == [folder1] + assert stored_obj.active_folders == [folder1] + assert stored_obj.library_root == [library] + assert stored_obj.actions == [library_folder_permission] + # use identity equality instread of object equality. + assert stored_obj.datasets[0].id == library_dataset.id + assert stored_obj.active_datasets[0].id == library_dataset.id def test_LibraryFolderPermissions(model, session, library_folder, role): From 5b284252567c1bc05cee49d67536471a5da57a24 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 7 Jul 2021 15:40:15 -0400 Subject: [PATCH 106/225] Automate class name extraction --- test/unit/model/test_model.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 59dd0df39f5..2a7ef08cae0 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1085,16 +1085,14 @@ def test_LibraryDatasetPermissions(model, session, library_dataset, role): class TestLibraryFolder: - @pytest.fixture - def cls_(self, model): - return model.LibraryFolder + def test_table(self, model): + cls = get_class(model, self) + assert cls.__tablename__ == 'library_folder' + assert has_index(cls.__table__, ('name',)) - def test_table(self, cls_, model): - assert cls_.__tablename__ == 'library_folder' - assert has_index(cls_.__table__, ('name',)) - - def test_columns(self, cls_, model, session, library_folder): - with dbcleanup(session, cls_): + def test_columns(self, model, session, library_folder): + cls = get_class(model, self) + with dbcleanup(session, cls): parent = library_folder create_time = datetime.now() update_time = create_time + timedelta(hours=1) @@ -1106,7 +1104,7 @@ class TestLibraryFolder: purged = False genome_build = 'c' - obj = cls_() + obj = cls() obj.parent = parent obj.create_time = create_time obj.update_time = update_time @@ -1119,7 +1117,7 @@ class TestLibraryFolder: obj.genome_build = genome_build obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls_, obj_id) + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.id == obj_id assert stored_obj.parent_id == parent.id assert stored_obj.create_time == create_time @@ -1133,11 +1131,12 @@ class TestLibraryFolder: assert stored_obj.genome_build == genome_build def test_relationships( - self, cls_, model, session, library_folder, library_dataset, library, library_folder_permission): - with dbcleanup(session, cls_): + self, model, session, library_folder, library_dataset, library, library_folder_permission): + cls = get_class(model, self) + with dbcleanup(session, cls): folder1 = model.LibraryFolder() - obj = cls_() + obj = cls() # no back reference, so dataset does not update folder; so we have to flush to the db # todo: ..but why is there no back reference? library_dataset.folder = obj @@ -1149,7 +1148,7 @@ class TestLibraryFolder: obj.actions.append(library_folder_permission) obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls_, obj_id) + stored_obj = get_stored_obj(session, cls, obj_id) assert stored_obj.parent.id == library_folder.id assert stored_obj.folders == [folder1] assert stored_obj.active_folders == [folder1] @@ -2558,3 +2557,9 @@ def get_stored_obj(session, cls, obj_id=None, stmt=None): def get_random_string(): """Generate unique values to accommodate unique constraints.""" return str(random.random()) + + +def get_class(model, test_class_instance): + prefix = len('Test') + class_name = test_class_instance.__class__.__name__[prefix:] + return getattr(model, class_name) From 7e21c8646b3dc5f3e91fee41e69cc5e5a8f0db75 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 7 Jul 2021 18:04:06 -0400 Subject: [PATCH 107/225] Improve class name extraction reuse across tests --- test/unit/model/test_model.py | 46 +++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 2a7ef08cae0..f3e35b36004 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -12,6 +12,19 @@ from sqlalchemy import ( import galaxy.model.mapping as mapping +class BaseTest: + @pytest.fixture + def cls_(self, model): + """ + Return class under test. + Assumptions: if the class under test is Foo, then the class grouping + the tests should be a subclass of BaseTest, named TestFoo. + """ + prefix = len('Test') + class_name = self.__class__.__name__[prefix:] + return getattr(model, class_name) + + def test_APIKeys(model, session, user): cls = model.APIKeys assert cls.__tablename__ == 'api_keys' @@ -1083,16 +1096,14 @@ def test_LibraryDatasetPermissions(model, session, library_dataset, role): assert stored_obj.role.id == role.id -class TestLibraryFolder: +class TestLibraryFolder(BaseTest): - def test_table(self, model): - cls = get_class(model, self) - assert cls.__tablename__ == 'library_folder' - assert has_index(cls.__table__, ('name',)) + def test_table(self, cls_, model): + assert cls_.__tablename__ == 'library_folder' + assert has_index(cls_.__table__, ('name',)) - def test_columns(self, model, session, library_folder): - cls = get_class(model, self) - with dbcleanup(session, cls): + def test_columns(self, cls_, model, session, library_folder): + with dbcleanup(session, cls_): parent = library_folder create_time = datetime.now() update_time = create_time + timedelta(hours=1) @@ -1104,7 +1115,7 @@ class TestLibraryFolder: purged = False genome_build = 'c' - obj = cls() + obj = cls_() obj.parent = parent obj.create_time = create_time obj.update_time = update_time @@ -1117,7 +1128,7 @@ class TestLibraryFolder: obj.genome_build = genome_build obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) + stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id assert stored_obj.parent_id == parent.id assert stored_obj.create_time == create_time @@ -1131,12 +1142,11 @@ class TestLibraryFolder: assert stored_obj.genome_build == genome_build def test_relationships( - self, model, session, library_folder, library_dataset, library, library_folder_permission): - cls = get_class(model, self) - with dbcleanup(session, cls): + self, cls_, model, session, library_folder, library_dataset, library, library_folder_permission): + with dbcleanup(session, cls_): folder1 = model.LibraryFolder() - obj = cls() + obj = cls_() # no back reference, so dataset does not update folder; so we have to flush to the db # todo: ..but why is there no back reference? library_dataset.folder = obj @@ -1148,7 +1158,7 @@ class TestLibraryFolder: obj.actions.append(library_folder_permission) obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) + stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == library_folder.id assert stored_obj.folders == [folder1] assert stored_obj.active_folders == [folder1] @@ -2557,9 +2567,3 @@ def get_stored_obj(session, cls, obj_id=None, stmt=None): def get_random_string(): """Generate unique values to accommodate unique constraints.""" return str(random.random()) - - -def get_class(model, test_class_instance): - prefix = len('Test') - class_name = test_class_instance.__class__.__name__[prefix:] - return getattr(model, class_name) From 24e0d99a13b8fbf55be6d7a32fd4679d8ffc027e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 8 Jul 2021 16:14:36 -0400 Subject: [PATCH 108/225] Map LibraryDataset declaratively (see note) If we have: `Column('info', Sting, key='_info')` then we remap like this: `_info = Column('info', String)` Because the key attribute will be overridden. In both cases, we have access to LibraryDataset._info (assuming it's an instance of LibraryDataset). In the case of LibraryDataset, if we were to do this: `info = Column('info', String, key='_info')`, which is the intuitive way to do it, then we'd get an error when we try to access LibraryDataset.info BEFORE explicitly setting it. `_info` is NOT created by the mapper, so when we access the property `info`, the code first checks the value of `self._info` - which will raise an Attribute Error. --- lib/galaxy/model/__init__.py | 34 ++++++++++++++++++- lib/galaxy/model/mapping.py | 36 --------------------- test/unit/model/test_model.py | 61 +++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 41c20fe9948..8b06c95546f 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4294,7 +4294,39 @@ class LibraryFolder(Base, Dictifiable, HasName, RepresentById): return f.library_root[0] -class LibraryDataset(RepresentById): +class LibraryDataset(Base, RepresentById): + __tablename__ = 'library_dataset' + + id = Column('id', Integer, primary_key=True) + # current version of dataset, if null, there is not a current version selected + library_dataset_dataset_association_id = Column('library_dataset_dataset_association_id', Integer, + ForeignKey('library_dataset_dataset_association.id', use_alter=True, name='library_dataset_dataset_association_id_fk'), + nullable=True, index=True) + folder_id = Column('folder_id', Integer, ForeignKey('library_folder.id'), index=True) + # not currently being used, but for possible future use + order_id = Column('order_id', Integer) + create_time = Column('create_time', DateTime, default=now) + update_time = Column('update_time', DateTime, default=now, onupdate=now) + # when not None/null this will supercede display in library (but not when imported into user's history?) + _name = Column('name', TrimmedString(255), index=True) + # when not None/null this will supercede display in library (but not when imported into user's history?) + _info = Column('info', TrimmedString(255)) + deleted = Column('deleted', Boolean, index=True, default=False) + purged = Column('purged', Boolean, index=True, default=False) + folder = relationship('LibraryFolder') + library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', + foreign_keys=library_dataset_dataset_association_id, + post_update=True) + expired_datasets = relationship('LibraryDatasetDatasetAssociation', + foreign_keys=[id, library_dataset_dataset_association_id], + primaryjoin=( + 'and_(LibraryDataset.id == LibraryDatasetDatasetAssociation.library_dataset_id, \ + not_(LibraryDataset.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.id))' + ), + viewonly=True, + uselist=True) + actions = relationship('LibraryDatasetPermissions', back_populates='library_dataset') + # This class acts as a proxy to the currently selected LDDA upload_options = [('upload_file', 'Upload files'), ('upload_directory', 'Upload directory of files'), diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 8af78968233..6b1a4a2ca17 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -193,25 +193,6 @@ model.UserRoleAssociation.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now)) -model.LibraryDataset.table = Table( - "library_dataset", metadata, - Column("id", Integer, primary_key=True), - # current version of dataset, if null, there is not a current version selected - Column("library_dataset_dataset_association_id", Integer, - ForeignKey("library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk"), - nullable=True, index=True), - Column("folder_id", Integer, ForeignKey("library_folder.id"), index=True), - # not currently being used, but for possible future use - Column("order_id", Integer), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - # when not None/null this will supercede display in library (but not when imported into user's history?) - Column("name", TrimmedString(255), key="_name", index=True), - # when not None/null this will supercede display in library (but not when imported into user's history?) - Column("info", TrimmedString(255), key="_info"), - Column("deleted", Boolean, index=True, default=False), - Column("purged", Boolean, index=True, default=False)) - model.LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_association", metadata, Column("id", Integer, primary_key=True), @@ -1135,23 +1116,6 @@ mapper_registry.map_imperatively(model.LibraryFolderInfoAssociation, model.Libra primaryjoin=(model.LibraryFolderInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -mapper_registry.map_imperatively(model.LibraryDataset, model.LibraryDataset.table, properties=dict( - folder=relation(model.LibraryFolder), - library_dataset_dataset_association=relation(model.LibraryDatasetDatasetAssociation, - foreign_keys=model.LibraryDataset.table.c.library_dataset_dataset_association_id, - post_update=True), - expired_datasets=relation(model.LibraryDatasetDatasetAssociation, - foreign_keys=[model.LibraryDataset.table.c.id, model.LibraryDataset.table.c.library_dataset_dataset_association_id], - primaryjoin=( - (model.LibraryDataset.table.c.id == model.LibraryDatasetDatasetAssociation.table.c.library_dataset_id) - & (not_(model.LibraryDataset.table.c.library_dataset_dataset_association_id - == model.LibraryDatasetDatasetAssociation.table.c.id)) - ), - viewonly=True, - uselist=True), - actions=relation(model.LibraryDatasetPermissions, back_populates='library_dataset'), -)) - mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssociation.table, properties=dict( dataset=relation(model.Dataset, primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.dataset_id == model.Dataset.table.c.id), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index f3e35b36004..e151c7e6b6e 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -899,6 +899,67 @@ def test_JobToOutputDatasetCollectionAssociation( assert stored_obj.dataset_collection_instance.id == history_dataset_collection_association.id +class TestLibraryDataset(BaseTest): + + def test_table(self, cls_, model): + assert cls_.__tablename__ == 'library_dataset' + + def test_columns( + self, cls_, model, session, library_dataset_dataset_association, library_folder): + with dbcleanup(session, cls_): + folder = library_folder + order_id = 9 + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + name = 'a' + info = 'b' + deleted = False + purged = False + + obj = cls_() + obj.folder = folder + obj.order_id = order_id + obj.create_time = create_time + obj.update_time = update_time + obj.name = name + obj.info = info + obj.deleted = deleted + obj.purged = purged + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.folder_id == folder.id + assert stored_obj.order_id == order_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.info == info + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + + def test_relationships(self, cls_, model, session, library_dataset_dataset_association, + library_folder, library_dataset_permission): + with dbcleanup(session, cls_): + folder = library_folder + obj = cls_() + obj.library_dataset_dataset_association = library_dataset_dataset_association + obj.folder = folder + + ldda = model.LibraryDatasetDatasetAssociation() + ldda.library_dataset = obj + persist(session, ldda) + + obj.actions.append(library_dataset_permission) + obj_id = persist(session, obj) + + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert stored_obj.folder.id == folder.id + assert stored_obj.expired_datasets[0].id == ldda.id + assert stored_obj.actions == [library_dataset_permission] + + def test_JobToOutputLibraryDatasetAssociation(model, session, library_dataset_dataset_association, job): cls = model.JobToOutputLibraryDatasetAssociation assert cls.__tablename__ == 'job_to_output_library_dataset' From 0c17df9f607aed2dcfccde54713c84ce9152325a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 19 Jul 2021 17:01:25 -0400 Subject: [PATCH 109/225] Rewrite all tests (see note) 1. Group model tests by classes. 2. Separate tests for each model into table-level, column, and relationship tests. 3. Ensure each object added to the database is deleted from the database upon test completion. Previous implementation deleted all objects of a given class, which may have caused conflicts when executing tests in parallel. In the updated version, only specific object are deleted. This is applied to objects added to the database in the body of a test, as well as fixtures. 4. Refactor, simplify helper functions. 5. Add draft documentation at the top of the module. --- test/unit/model/test_model.py | 3596 +++++++++++++++++++-------------- 1 file changed, 2058 insertions(+), 1538 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index e151c7e6b6e..8a5e785e9ef 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1,9 +1,54 @@ +""" +TODO: documentaion on writing model tests goes here + +Example: + +TestFoo(BaseTest): # BaseTest is parent class + + # test table-level properties (e.g. table name, indexes, unique constraints) + def test_table(self, cls_): # cls_ is fixture defined in parent class, returns class under test. + assert cls_.__tablename__ == 'foo' # assert table name + assert has_index(cls.__table__, ('foo',)) # second arg is a tuple containg field names + + # test column-mapped fields + def test_columns(self, session, cls_): + some_foo, some_bar = 42, 'stuff' # create test values here + obj = cls_(foo=some_foo) # pass test values to constructor + obj.bar = some_bar # assign test values to obj if can't pass to constructor + + with dbcleanup(session, obj) as obj_id: # use context manager to ensure obj is deleted from db on exit. + stored_obj = get_stored_obj(session, cls_, obj_id) # retrieve data from db and create new obj. + # check ALL column-mapped fields + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.user_id == user_id + assert stored_obj.key == key + + # test relationship-mapped fields + def test_relationships(self, session, cls_): + obj = cls_() # use minimal possible constructor + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + # check ALL relationship-mapped fields + +# TODO: ADD MORE EXAMPLES! +- point to TestLibraryDataset for example of using more than one object of same type (and, thus, + not being able to use a fixture. +- TestPage: how to call cleanup explicitly (see average_ratings) +# TODO: explain why we test for columns: + assert stored_obj.user_id == user_id + and for relationships: + assert stored_obj.user.id == user_id +""" + import random from contextlib import contextmanager from datetime import datetime, timedelta import pytest from sqlalchemy import ( + and_, delete, select, UniqueConstraint, @@ -25,240 +70,266 @@ class BaseTest: return getattr(model, class_name) -def test_APIKeys(model, session, user): - cls = model.APIKeys - assert cls.__tablename__ == 'api_keys' - with dbcleanup(session, cls): +class TestAPIKeys(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'api_keys' + + def test_columns(self, session, cls_, user): create_time, user_id, key = datetime.now(), user.id, get_random_string() - obj = cls(user_id=user_id, key=key, create_time=create_time) - obj_id = persist(session, obj) + obj = cls_(user_id=user_id, key=key, create_time=create_time) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.user_id == user_id - assert stored_obj.key == key - # test mapped relationships - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.user_id == user_id + assert stored_obj.key == key + + def test_relationships(self, session, cls_, user): + obj = cls_(user_id=user.id, key=get_random_string()) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id -def test_CloudAuthz(model, session, user, user_authnz_token): - cls = model.CloudAuthz - assert cls.__tablename__ == 'cloudauthz' - with dbcleanup(session, cls): +class TestCloudAuthz(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'cloudauthz' + + def test_columns(self, session, cls_, user, user_authnz_token): provider, config, description = 'a', 'b', 'c' - obj = cls(user.id, provider, config, user_authnz_token.id, description) - obj_id = persist(session, obj) + obj = cls_(user.id, provider, config, user_authnz_token.id, description) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.user_id == user.id - assert stored_obj.provider == provider - assert stored_obj.config == config - assert stored_obj.authn_id == user_authnz_token.id - assert stored_obj.tokens is None - assert stored_obj.last_update - assert stored_obj.last_activity - assert stored_obj.description == description - assert stored_obj.create_time - # test mapped relationships - assert stored_obj.user.id == user.id - assert stored_obj.authn.id == user_authnz_token.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user_id == user.id + assert stored_obj.provider == provider + assert stored_obj.config == config + assert stored_obj.authn_id == user_authnz_token.id + assert stored_obj.tokens is None + assert stored_obj.last_update + assert stored_obj.last_activity + assert stored_obj.description == description + assert stored_obj.create_time + + def test_relationships(self, session, cls_, user, user_authnz_token): + obj = cls_(user.id, None, None, user_authnz_token.id, 'c') + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id + assert stored_obj.authn.id == user_authnz_token.id -def test_CustosAuthnzToken(model, session, user): - cls = model.CustosAuthnzToken - assert cls.__tablename__ == 'custos_authnz_token' - assert has_unique_constraint(cls.__table__, ('user_id', 'external_user_id', 'provider')) - assert has_unique_constraint(cls.__table__, ('external_user_id', 'provider')) - with dbcleanup(session, cls): - external_user_id = get_random_string() - provider = get_random_string() - access_token = 'c' - id_token = 'd' - refresh_token = 'e' - expiration_time = datetime.now() - refresh_expiration_time = datetime.now() - obj = cls(user, external_user_id, provider, access_token, id_token, refresh_token, - expiration_time, refresh_expiration_time) - obj_id = persist(session, obj) +class TestDatasetHash(BaseTest): - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.user_id == user.id - assert stored_obj.external_user_id == external_user_id - assert stored_obj.provider == provider - assert stored_obj.access_token == access_token - assert stored_obj.id_token == id_token - assert stored_obj.refresh_token == refresh_token - assert stored_obj.expiration_time == expiration_time - assert stored_obj.refresh_expiration_time == refresh_expiration_time - # test mapped relationships - assert stored_obj.user.id == user.id + def test_table(self, cls_): + assert cls_.__tablename__ == 'dataset_hash' - -def test_DatasetHash(model, session, dataset): - cls = model.DatasetHash - assert cls.__tablename__ == 'dataset_hash' - with dbcleanup(session, cls): + def test_columns(self, session, cls_, dataset): hash_function, hash_value, extra_files_path = 'a', 'b', 'c' - obj = cls() + obj = cls_() obj.dataset = dataset obj.hash_function = hash_function obj.hash_value = hash_value obj.extra_files_path = extra_files_path - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.dataset_id == dataset.id - assert stored_obj.hash_function == hash_function - assert stored_obj.hash_value == hash_value - assert stored_obj.extra_files_path == extra_files_path - # test mapped relationships - assert stored_obj.dataset.id == dataset.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.dataset_id == dataset.id + assert stored_obj.hash_function == hash_function + assert stored_obj.hash_value == hash_value + assert stored_obj.extra_files_path == extra_files_path + + def test_relationships(self, session, cls_, dataset): + obj = cls_() + obj.dataset = dataset + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset.id == dataset.id -def test_DatasetSource(model, session, dataset, dataset_source_hash): - cls = model.DatasetSource - assert cls.__tablename__ == 'dataset_source' - with dbcleanup(session, cls): +class TestDatasetSource(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'dataset_source' + + def test_columns(self, session, cls_, dataset, dataset_source_hash): source_uri, extra_files_path, transform = 'a', 'b', 'c' - obj = cls() + obj = cls_() obj.dataset = dataset obj.source_uri = source_uri obj.extra_files_path = extra_files_path obj.transform = transform + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.dataset_id == dataset.id + assert stored_obj.source_uri == source_uri + assert stored_obj.extra_files_path == extra_files_path + assert stored_obj.transform == transform + + def test_relationships(self, session, cls_, dataset, dataset_source_hash): + obj = cls_() + obj.dataset = dataset obj.hashes.append(dataset_source_hash) - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.dataset_id == dataset.id - assert stored_obj.source_uri == source_uri - assert stored_obj.extra_files_path == extra_files_path - assert stored_obj.transform == transform - # test mapped relationships - assert stored_obj.dataset.id == dataset.id - assert stored_obj.hashes == [dataset_source_hash] + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset.id == dataset.id + assert stored_obj.hashes == [dataset_source_hash] -def test_DatasetSourceHash(model, session, dataset_source): - cls = model.DatasetSourceHash - assert cls.__tablename__ == 'dataset_source_hash' - with dbcleanup(session, cls): +class TestDatasetSourceHash(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'dataset_source_hash' + + def test_columns(self, session, cls_, dataset_source): hash_function, hash_value = 'a', 'b' - obj = cls() + obj = cls_() obj.source = dataset_source obj.hash_function = hash_function obj.hash_value = hash_value - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.dataset_source_id == dataset_source.id - assert stored_obj.hash_function == hash_function - assert stored_obj.hash_value == hash_value - # test mapped relationships - assert stored_obj.source.id == dataset_source.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.dataset_source_id == dataset_source.id + assert stored_obj.hash_function == hash_function + assert stored_obj.hash_value == hash_value + + def test_relationships(self, session, cls_, dataset_source): + obj = cls_() + obj.source = dataset_source + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.source.id == dataset_source.id -def test_DatasetPermissions(model, session, dataset, role): - cls = model.DatasetPermissions - assert cls.__tablename__ == 'dataset_permissions' - with dbcleanup(session, cls): +class TestDatasetPermissions(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'dataset_permissions' + + def test_columns(self, session, cls_, dataset, role): action = 'a' - obj = cls(action, dataset, role) create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(action, dataset, role) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.action == action - assert stored_obj.dataset_id == dataset.id - assert stored_obj.role_id == role.id - # test mapped relationships - assert stored_obj.dataset == dataset - assert stored_obj.role == role + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.update_time == update_time + assert stored_obj.action == action + assert stored_obj.dataset_id == dataset.id + assert stored_obj.role_id == role.id + + def test_relationships(self, session, cls_, dataset, role): + obj = cls_(None, dataset, role) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset == dataset + assert stored_obj.role == role -def test_DefaultHistoryPermissions(model, session, history, role): - cls = model.DefaultHistoryPermissions - assert cls.__tablename__ == 'default_history_permissions' - with dbcleanup(session, cls): +class TestDefaultHistoryPermissions(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'default_history_permissions' + + def test_columns(self, session, cls_, history, role): action = 'a' - obj = cls(history, action, role) - obj_id = persist(session, obj) + obj = cls_(history, action, role) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.action == action - assert stored_obj.history_id == history.id - assert stored_obj.role_id == role.id - # test mapped relationships - assert stored_obj.history.id == history.id - assert stored_obj.role == role + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.action == action + assert stored_obj.history_id == history.id + assert stored_obj.role_id == role.id + + def test_relationships(self, session, cls_, history, role): + obj = cls_(history, None, role) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history.id == history.id + assert stored_obj.role == role -def test_DefaultQuotaAssociation(model, session, quota): - cls = model.DefaultQuotaAssociation - assert cls.__tablename__ == 'default_quota_association' - with dbcleanup(session, cls): - type = cls.types.REGISTERED - obj = cls(type, quota) +class TestDefaultQuotaAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'default_quota_association' + + def test_columns(self, session, cls_, quota): + type_ = cls_.types.REGISTERED create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(type_, quota) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.type == type - assert stored_obj.quota_id == quota.id - # test mapped relationships - assert stored_obj.quota.id == quota.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.type == type_ + assert stored_obj.quota_id == quota.id + + def test_relationships(self, session, cls_, quota): + obj = cls_(cls_.types.REGISTERED, quota) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.quota.id == quota.id -def test_DefaultUserPermissions(model, session, user, role): - cls = model.DefaultUserPermissions - assert cls.__tablename__ == 'default_user_permissions' - with dbcleanup(session, cls): +class TestDefaultUserPermissions(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'default_user_permissions' + + def test_columns(self, session, cls_, user, role): action = 'a' - obj = cls(user, action, role) - obj_id = persist(session, obj) + obj = cls_(user, action, role) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.user_id == user.id - assert stored_obj.action == action - assert stored_obj.role_id == role.id - # test mapped relationships - assert stored_obj.user.id == user.id - assert stored_obj.role.id == role.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user_id == user.id + assert stored_obj.action == action + assert stored_obj.role_id == role.id + + def test_relationships(self, session, cls_, user, role): + obj = cls_(user, None, role) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id + assert stored_obj.role.id == role.id -def test_DynamicTool(model, session, workflow_step): - cls = model.DynamicTool - assert cls.__tablename__ == 'dynamic_tool' - with dbcleanup(session, cls): +class TestDynamicTool(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'dynamic_tool' + + def test_columns(self, session, cls_): tool_format = 'a' tool_id = 'b' tool_version = 'c' @@ -268,668 +339,824 @@ def test_DynamicTool(model, session, workflow_step): active = True hidden = True value = 'f' - obj = cls(tool_format, tool_id, tool_version, tool_path, tool_directory, uuid, - active, hidden, value) create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(tool_format, tool_id, tool_version, tool_path, tool_directory, uuid, + active, hidden, value) obj.create_time = create_time obj.update_time = update_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.uuid + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.tool_id == tool_id + assert stored_obj.tool_version == tool_version + assert stored_obj.tool_format == tool_format + assert stored_obj.tool_path == tool_path + assert stored_obj.tool_directory == tool_directory + assert stored_obj.hidden == hidden + assert stored_obj.active == active + assert stored_obj.value == value + + def test_relationships(self, session, cls_, workflow_step): + obj = cls_() obj.workflow_steps.append(workflow_step) - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.uuid - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.tool_id == tool_id - assert stored_obj.tool_version == tool_version - assert stored_obj.tool_format == tool_format - assert stored_obj.tool_path == tool_path - assert stored_obj.tool_directory == tool_directory - assert stored_obj.hidden == hidden - assert stored_obj.active == active - assert stored_obj.value == value - # test mapped relationships - assert workflow_step in stored_obj.workflow_steps + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_steps == [workflow_step] -def test_FormDefinition(model, session, form_definition_current): - cls = model.FormDefinition - assert cls.__tablename__ == 'form_definition' - with dbcleanup(session, cls): +class TestFormDefinition(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'form_definition' + + def test_columns(self, session, cls_, form_definition_current): name, desc, fields, type, layout = 'a', 'b', 'c', 'd', 'e' - obj = cls() + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj = cls_() obj.name = name obj.desc = desc obj.form_definition_current = form_definition_current obj.fields = fields obj.type = type obj.layout = layout - create_time = datetime.now() - update_time = create_time + timedelta(hours=1) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.name == name - assert stored_obj.desc == desc - assert stored_obj.form_definition_current_id == form_definition_current.id - assert stored_obj.fields == fields - assert stored_obj.type == type - assert stored_obj.layout == layout - # test mapped relationships - assert stored_obj.form_definition_current.id == form_definition_current.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.desc == desc + assert stored_obj.form_definition_current_id == form_definition_current.id + assert stored_obj.fields == fields + assert stored_obj.type == type + assert stored_obj.layout == layout + + def test_relationships(self, session, cls_, form_definition_current): + obj = cls_(name='a', form_definition_current=form_definition_current) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.form_definition_current.id == form_definition_current.id -def test_FormDefinitionCurrent(model, session, form_definition): - cls = model.FormDefinitionCurrent - assert cls.__tablename__ == 'form_definition_current' - with dbcleanup(session, cls): - obj = cls() +class TestFormDefinitionCurrent(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'form_definition_current' + + def test_columns(self, session, cls_, form_definition): create_time = datetime.now() update_time = create_time + timedelta(hours=1) deleted = True + obj = cls_() obj.create_time = create_time obj.update_time = update_time obj.latest_form = form_definition obj.deleted = deleted + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.latest_form_id == form_definition.id + assert stored_obj.deleted == deleted + + def test_relationships(self, session, cls_, form_definition): + obj = cls_(form_definition) obj.forms.append(form_definition) - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.latest_form_id == form_definition.id - assert stored_obj.deleted == deleted - # test mapped relationships - assert stored_obj.latest_form.id == form_definition.id - assert stored_obj.forms == [form_definition] + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.latest_form.id == form_definition.id + assert stored_obj.forms == [form_definition] -def test_FormValues(model, session, form_definition): - cls = model.FormValues - assert cls.__tablename__ == 'form_values' - with dbcleanup(session, cls): +class TestFormValues(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'form_values' + + def test_columns(self, session, cls_, form_definition): content = 'a' - obj = cls() - obj.form_definition = form_definition create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_() + obj.form_definition = form_definition obj.content = content obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.content == content - assert stored_obj.form_definition_id == form_definition.id - # test mapped relationships - assert stored_obj.form_definition.id == form_definition.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.content == content + assert stored_obj.form_definition_id == form_definition.id + + def test_relationships(self, session, cls_, form_definition): + obj = cls_(form_definition) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.form_definition.id == form_definition.id -def test_GroupQuotaAssociation(model, session, group, quota): - cls = model.GroupQuotaAssociation - assert cls.__tablename__ == 'group_quota_association' - with dbcleanup(session, cls): - obj = cls(group, quota) +class TestGroupQuotaAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'group_quota_association' + + def test_columns(self, session, cls_, group, quota): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj = cls_(group, quota) + obj.create_time = create_time + obj.update_time = update_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.group_id == group.id + assert stored_obj.quota_id == quota.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + + def test_relationships(self, session, cls_, group, quota): + obj = cls_(group, quota) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.group.id == group.id + assert stored_obj.quota.id == quota.id + + +class TestGroupRoleAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'group_role_association' + + def test_columns(self, session, cls_, group, role): + obj = cls_(group, role) create_time = datetime.now() update_time = create_time + timedelta(hours=1) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.group_id == group.id - assert stored_obj.quota_id == quota.id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - # test mapped relationships - assert stored_obj.group.id == group.id - assert stored_obj.quota.id == quota.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.group_id == group.id + assert stored_obj.role_id == role.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + + def test_relationships(self, session, cls_, group, role): + obj = cls_(group, role) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.group.id == group.id + assert stored_obj.role.id == role.id -def test_GroupRoleAssociation(model, session, group, role): - cls = model.GroupRoleAssociation - assert cls.__tablename__ == 'group_role_association' - with dbcleanup(session, cls): - obj = cls(group, role) - create_time = datetime.now() - update_time = create_time + timedelta(hours=1) - obj.create_time = create_time - obj.update_time = update_time - obj_id = persist(session, obj) +class TestHistoryUserShareAssociation(BaseTest): - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.group_id == group.id - assert stored_obj.role_id == role.id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - # test mapped relationships - assert stored_obj.group.id == group.id - assert stored_obj.role.id == role.id + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_user_share_association' - -def test_HistoryUserShareAssociation(model, session, history, user): - cls = model.HistoryUserShareAssociation - assert cls.__tablename__ == 'history_user_share_association' - with dbcleanup(session, cls): - obj = cls() + def test_columns(self, session, cls_, history, user): + obj = cls_() obj.history = history obj.user = user - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.history_id == history.id - assert stored_obj.user_id == user.id - # test mapped relationships - assert stored_obj.history_id == history.id - assert stored_obj.user_id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id + + def test_relationships(self, session, cls_, history, user): + obj = cls_() + obj.history = history + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id -def test_HistoryAudit(model, session, history): - cls = model.HistoryAudit - assert cls.__tablename__ == 'history_audit' - with dbcleanup(session, cls): +class TestHistoryAudit(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_audit' + + def test_columns_and_relationships(self, session, cls_, history): update_time = datetime.now() - obj = cls(history, update_time) - persist(session, obj, return_id=False) + obj = cls_(history, update_time) - stmt = select(cls).where(cls.history_id == history.id, cls.update_time == update_time) - stored_obj = get_stored_obj(session, cls, stmt=stmt) - # test mapped columns - assert stored_obj.history_id == history.id - assert stored_obj.update_time == update_time - # test mapped relationships - assert stored_obj.history.id == history.id + where_clause = and_(cls_.history_id == history.id, cls_.update_time == update_time) + + with dbcleanup(session, obj, where_clause): + stored_obj = get_stored_obj2(session, cls_, where_clause=where_clause) + # test columns + assert stored_obj.history_id == history.id + assert stored_obj.update_time == update_time + # test relationships + assert stored_obj.history.id == history.id -def test_HistoryAnnotationAssociation(model, session, history, user): - cls = model.HistoryAnnotationAssociation - assert cls.__tablename__ == 'history_annotation_association' - assert has_index(cls.__table__, ('annotation',)) - with dbcleanup(session, cls): +class TestHistoryAnnotationAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_annotation_association' + assert has_index(cls_.__table__, ('annotation',)) + + def test_columns(self, session, cls_, history, user): annotation = 'a' - obj = cls() + obj = cls_() obj.user = user obj.history = history obj.annotation = annotation - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.history_id == history.id - assert stored_obj.user_id == user.id - assert stored_obj.annotation == annotation - # test mapped relationships - assert stored_obj.history.id == history.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id + assert stored_obj.annotation == annotation + + def test_relationships(self, session, cls_, history, user): + obj = cls_() + obj.user = user + obj.history = history + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history.id == history.id + assert stored_obj.user.id == user.id -def test_HistoryDatasetAssociationAnnotationAssociation( - model, session, history_dataset_association, user): - cls = model.HistoryDatasetAssociationAnnotationAssociation - assert cls.__tablename__ == 'history_dataset_association_annotation_association' - assert has_index(cls.__table__, ('annotation',)) - with dbcleanup(session, cls): +class TestHistoryDatasetAssociationAnnotationAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_association_annotation_association' + assert has_index(cls_.__table__, ('annotation',)) + + def test_columns(self, session, cls_, history_dataset_association, user): annotation = 'a' - obj = cls() + obj = cls_() obj.user = user obj.hda = history_dataset_association obj.annotation = annotation - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.history_dataset_association_id == history_dataset_association.id - assert stored_obj.user_id == user.id - assert stored_obj.annotation == annotation - # test mapped relationships - assert stored_obj.hda.id == history_dataset_association.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.user_id == user.id + assert stored_obj.annotation == annotation + + def test_relationships(self, session, cls_, history_dataset_association, user): + obj = cls_() + obj.user = user + obj.hda = history_dataset_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.hda.id == history_dataset_association.id + assert stored_obj.user.id == user.id -def test_HistoryDatasetAssociationRatingAssociation(model, session, history_dataset_association, user): - cls = model.HistoryDatasetAssociationRatingAssociation - assert cls.__tablename__ == 'history_dataset_association_rating_association' - with dbcleanup(session, cls): +class TestHistoryDatasetAssociationRatingAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_association_rating_association' + + def test_columns(self, session, cls_, history_dataset_association, user): rating = 9 - obj = cls(user, history_dataset_association, rating) - obj_id = persist(session, obj) + obj = cls_(user, history_dataset_association, rating) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.history_dataset_association_id == history_dataset_association.id - assert stored_obj.user_id == user.id - assert stored_obj.rating == rating - # test mapped relationships - assert stored_obj.history_dataset_association.id == history_dataset_association.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user_id == user.id + assert stored_obj.rating == rating + + def test_relationships(self, session, cls_, history_dataset_association, user): + obj = cls_(user, history_dataset_association, 1) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history_dataset_association.id == history_dataset_association.id + assert stored_obj.user.id == user.id -def test_HistoryDatasetAssociationTagAssociation( - model, session, history_dataset_association, tag, user): - cls = model.HistoryDatasetAssociationTagAssociation - assert cls.__tablename__ == 'history_dataset_association_tag_association' - with dbcleanup(session, cls): +class TestHistoryDatasetAssociationTagAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_association_tag_association' + + def test_columns(self, session, cls_, history_dataset_association, tag, user): user_tname, value, user_value = 'a', 'b', 'c' - obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) - obj.user_value = user_value + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value, user_value=user_value) obj.history_dataset_association = history_dataset_association - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.history_dataset_association_id == history_dataset_association.id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - # test mapped relationships - assert stored_obj.history_dataset_association.id == history_dataset_association.id - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, history_dataset_association, tag, user): + obj = cls_(user=user, tag=tag) + obj.history_dataset_association = history_dataset_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history_dataset_association.id == history_dataset_association.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id -def test_HistoryDatasetCollectionAssociationAnnotationAssociation( - model, session, history_dataset_collection_association, user): - cls = model.HistoryDatasetCollectionAssociationAnnotationAssociation - assert cls.__tablename__ == 'history_dataset_collection_annotation_association' - with dbcleanup(session, cls): +class TestHistoryDatasetCollectionAssociationAnnotationAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_collection_annotation_association' + + def test_columns(self, session, cls_, history_dataset_collection_association, user): annotation = 'a' - obj = cls() + obj = cls_() obj.user = user obj.history_dataset_collection = history_dataset_collection_association obj.annotation = annotation - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id - assert stored_obj.user_id == user.id - assert stored_obj.annotation == annotation - # test mapped relationships - assert stored_obj.history_dataset_collection.id == history_dataset_collection_association.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.user_id == user.id + assert stored_obj.annotation == annotation + + def test_relationships(self, session, cls_, history_dataset_collection_association, user): + obj = cls_() + obj.user = user + obj.history_dataset_collection = history_dataset_collection_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history_dataset_collection.id == history_dataset_collection_association.id + assert stored_obj.user.id == user.id -def test_HistoryDatasetCollectionRatingAssociation( - model, session, history_dataset_collection_association, user): - cls = model.HistoryDatasetCollectionRatingAssociation - assert cls.__tablename__ == 'history_dataset_collection_rating_association' - with dbcleanup(session, cls): +class TestHistoryDatasetCollectionRatingAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_collection_rating_association' + + def test_columns(self, session, cls_, history_dataset_collection_association, user): rating = 9 - obj = cls(user, history_dataset_collection_association, rating) - obj_id = persist(session, obj) + obj = cls_(user, history_dataset_collection_association, rating) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id - assert stored_obj.user.id == user.id - assert stored_obj.rating == rating - # test mapped relationships - assert stored_obj.dataset_collection.id == history_dataset_collection_association.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.user.id == user.id + assert stored_obj.rating == rating + + def test_relationships(self, session, cls_, history_dataset_collection_association, user): + obj = cls_(user, history_dataset_collection_association, 1) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id + assert stored_obj.user.id == user.id -def test_HistoryDatasetCollectionTagAssociation(model, session, history_dataset_collection_association, tag, user): - cls = model.HistoryDatasetCollectionTagAssociation - assert cls.__tablename__ == 'history_dataset_collection_tag_association' - with dbcleanup(session, cls): +class TestHistoryDatasetCollectionTagAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_collection_tag_association' + + def test_columns(self, session, cls_, history_dataset_collection_association, tag, user): user_tname, value, user_value = 'a', 'b', 'c' - obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) - obj.user_value = user_value + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value, user_value=user_value) obj.dataset_collection = history_dataset_collection_association - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - # test mapped relationships - assert stored_obj.dataset_collection.id == history_dataset_collection_association.id - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, history_dataset_collection_association, tag, user): + obj = cls_(user=user, tag=tag) + obj.dataset_collection = history_dataset_collection_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id -def test_HistoryRatingAssociation(model, session, history, user): - cls = model.HistoryRatingAssociation - assert cls.__tablename__ == 'history_rating_association' - with dbcleanup(session, cls): +class TestHistoryRatingAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_rating_association' + + def test_columns(self, session, cls_, history, user): rating = 9 - obj = cls(user, history, rating) - obj_id = persist(session, obj) + obj = cls_(user, history, rating) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.history_id == history.id - assert stored_obj.user_id == user.id - assert stored_obj.rating == rating - # test mapped relationships - assert stored_obj.history.id == history.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id + assert stored_obj.rating == rating + + def test_relationships(self, session, cls_, history, user): + obj = cls_(user, history, 1) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history.id == history.id + assert stored_obj.user.id == user.id -def test_HistoryTagAssociation(model, session, history, tag, user): - cls = model.HistoryTagAssociation - assert cls.__tablename__ == 'history_tag_association' +class TestHistoryTagAssociation(BaseTest): - with dbcleanup(session, cls): + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_tag_association' + + def test_columns(self, session, cls_, history, tag, user): user_tname, value, user_value = 'a', 'b', 'c' - obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) obj.user_value = user_value obj.history = history - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.history_id == history.id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - # test mapped relationships - assert stored_obj.history.id == history.id - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_id == history.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, history, tag, user): + obj = cls_(user=user, tag=tag) + obj.history = history + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history.id == history.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id -def test_JobMetricNumeric(model, session, job): - cls = model.JobMetricNumeric - assert cls.__tablename__ == 'job_metric_numeric' - with dbcleanup(session, cls): +class TestJobMetricNumeric(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_metric_numeric' + + def test_columns(self, session, cls_, job): plugin, metric_name, metric_value = 'a', 'b', 9 - obj = cls(plugin, metric_name, metric_value) + obj = cls_(plugin, metric_name, metric_value) obj.job = job - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.plugin == plugin - assert stored_obj.metric_value == metric_value - # test mapped relationships - assert stored_obj.job.id == job.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.plugin == plugin + assert stored_obj.metric_value == metric_value + + def test_relationships(self, session, cls_, job): + obj = cls_(None, None, None) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id -def test_JobMetricText(model, session, job): - cls = model.JobMetricText - assert cls.__tablename__ == 'job_metric_text' - with dbcleanup(session, cls): +class TestJobMetricText(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_metric_text' + + def test_columns(self, session, cls_, job): plugin, metric_name, metric_value = 'a', 'b', 'c' - obj = cls(plugin, metric_name, metric_value) + obj = cls_(plugin, metric_name, metric_value) obj.job = job - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.plugin == plugin - assert stored_obj.metric_value == metric_value - # test mapped relationships - assert stored_obj.job.id == job.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.plugin == plugin + assert stored_obj.metric_value == metric_value + + def test_relationships(self, session, cls_, job): + obj = cls_(None, None, None) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id -def test_JobParameter(model, session, job): - cls = model.JobParameter - assert cls.__tablename__ == 'job_parameter' - with dbcleanup(session, cls): +class TestJobParameter(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_parameter' + + def test_columns(self, session, cls_, job): name, value = 'a', 'b' - obj = cls(name, value) + obj = cls_(name, value) obj.job_id = job.id - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.name == name - assert stored_obj.value == value + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.name == name + assert stored_obj.value == value -def test_JobStateHistory(model, session, job): - cls = model.JobStateHistory - assert cls.__tablename__ == 'job_state_history' - with dbcleanup(session, cls): +class TestJobStateHistory(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_state_history' + + def test_columns(self, session, cls_, job): state, info = job.state, job.info - obj = cls(job) create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(job) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.job_id == job.id - assert stored_obj.state == state - assert stored_obj.info == info - # test mapped relationships + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.job_id == job.id + assert stored_obj.state == state + assert stored_obj.info == info + + def test_relationships(self, session, cls_, job): + obj = cls_(job) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.job.id == job.id -def test_JobToImplicitOutputDatasetCollectionAssociation( - model, session, dataset_collection, job): - cls = model.JobToImplicitOutputDatasetCollectionAssociation - assert cls.__tablename__ == 'job_to_implicit_output_dataset_collection' +class TestJobToImplicitOutputDatasetCollectionAssociation(BaseTest): - with dbcleanup(session, cls): + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_to_implicit_output_dataset_collection' + + def test_columns(self, session, cls_, dataset_collection, job): name = 'a' - obj = cls(name, dataset_collection) + obj = cls_(name, dataset_collection) obj.job = job - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.dataset_collection_id == dataset_collection.id - assert stored_obj.name == name - # test mapped relationships - assert stored_obj.job.id == job.id - assert stored_obj.dataset_collection.id == dataset_collection.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_collection_id == dataset_collection.id + assert stored_obj.name == name + + def test_relationships(self, session, cls_, dataset_collection, job): + obj = cls_(None, dataset_collection) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.dataset_collection.id == dataset_collection.id -def test_JobToInputDatasetCollectionAssociation( - model, session, history_dataset_collection_association, job): - cls = model.JobToInputDatasetCollectionAssociation - assert cls.__tablename__ == 'job_to_input_dataset_collection' +class TestJobToInputDatasetCollectionAssociation(BaseTest): - with dbcleanup(session, cls): + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_to_input_dataset_collection' + + def test_columns(self, session, cls_, history_dataset_collection_association, job): name = 'a' - obj = cls(name, history_dataset_collection_association) + obj = cls_(name, history_dataset_collection_association) obj.job = job - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.dataset_collection_id == history_dataset_collection_association.id - assert stored_obj.name == name - # test mapped relationships - assert stored_obj.job.id == job.id - assert stored_obj.dataset_collection.id == history_dataset_collection_association.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.name == name + + def test_relationships(self, session, cls_, history_dataset_collection_association, job): + obj = cls_(None, history_dataset_collection_association) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id -def test_JobToInputDatasetCollectionElementAssociation( - model, session, dataset_collection_element, job): - cls = model.JobToInputDatasetCollectionElementAssociation - assert cls.__tablename__ == 'job_to_input_dataset_collection_element' +class TestJobToInputDatasetCollectionElementAssociation(BaseTest): - with dbcleanup(session, cls): + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_to_input_dataset_collection_element' + + def test_columns(self, session, cls_, dataset_collection_element, job): name = 'a' - obj = cls(name, dataset_collection_element) + obj = cls_(name, dataset_collection_element) obj.job = job - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.dataset_collection_element_id == dataset_collection_element.id - assert stored_obj.name == name - # test mapped relationships - assert stored_obj.job.id == job.id - assert stored_obj.dataset_collection_element.id == dataset_collection_element.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_collection_element_id == dataset_collection_element.id + assert stored_obj.name == name + + def test_relationships(self, session, cls_, dataset_collection_element, job): + obj = cls_(None, dataset_collection_element) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.dataset_collection_element.id == dataset_collection_element.id -def test_JobToInputDatasetAssociation(model, session, history_dataset_association, job): - cls = model.JobToInputDatasetAssociation - assert cls.__tablename__ == 'job_to_input_dataset' +class TestJobToInputDatasetAssociation(BaseTest): - with dbcleanup(session, cls): + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_to_input_dataset' + + def test_columns(self, session, cls_, history_dataset_association, job): name, dataset_version = 'a', 9 - obj = cls(name, history_dataset_association) + obj = cls_(name, history_dataset_association) obj.job = job obj.dataset_version = dataset_version - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.dataset_id == history_dataset_association.id - assert stored_obj.dataset_version == dataset_version - assert stored_obj.name == name - # test mapped relationships - assert stored_obj.job.id == job.id - assert stored_obj.dataset.id == history_dataset_association.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_id == history_dataset_association.id + assert stored_obj.dataset_version == dataset_version + assert stored_obj.name == name - -def test_JobToInputLibraryDatasetAssociation( - model, session, library_dataset_dataset_association, job): - cls = model.JobToInputLibraryDatasetAssociation - assert cls.__tablename__ == 'job_to_input_library_dataset' - - with dbcleanup(session, cls): - name = 'a' - obj = cls(name, library_dataset_dataset_association) + def test_relationships(self, session, cls_, history_dataset_association, job): + obj = cls_(None, history_dataset_association) obj.job = job - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.ldda_id == library_dataset_dataset_association.id - assert stored_obj.name == name - # test mapped relationships - assert stored_obj.job.id == job.id - assert stored_obj.dataset.id == library_dataset_dataset_association.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.dataset.id == history_dataset_association.id -def test_JobToOutputDatasetAssociation(model, session, history_dataset_association, job): - cls = model.JobToOutputDatasetAssociation - assert cls.__tablename__ == 'job_to_output_dataset' +class TestJobToInputLibraryDatasetAssociation(BaseTest): - with dbcleanup(session, cls): + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_to_input_library_dataset' + + def test_columns(self, session, cls_, library_dataset_dataset_association, job): name = 'a' - obj = cls(name, history_dataset_association) + obj = cls_(name, library_dataset_dataset_association) obj.job = job - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.dataset_id == history_dataset_association.id - assert stored_obj.name == name - # test mapped relationships - assert stored_obj.job.id == job.id - assert stored_obj.dataset.id == history_dataset_association.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.ldda_id == library_dataset_dataset_association.id + assert stored_obj.name == name + + def test_relationships(self, session, cls_, library_dataset_dataset_association, job): + obj = cls_(None, library_dataset_dataset_association) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.dataset.id == library_dataset_dataset_association.id -def test_JobToOutputDatasetCollectionAssociation( - model, session, history_dataset_collection_association, job): - cls = model.JobToOutputDatasetCollectionAssociation - assert cls.__tablename__ == 'job_to_output_dataset_collection' +class TestJobToOutputDatasetAssociation(BaseTest): - with dbcleanup(session, cls): + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_to_output_dataset' + + def test_columns(self, session, cls_, history_dataset_association, job): name = 'a' - obj = cls(name, history_dataset_collection_association) + obj = cls_(name, history_dataset_association) obj.job = job - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.dataset_collection_id == history_dataset_collection_association.id - assert stored_obj.name == name - # test mapped relationships - assert stored_obj.job.id == job.id - assert stored_obj.dataset_collection_instance.id == history_dataset_collection_association.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_id == history_dataset_association.id + assert stored_obj.name == name + + def test_relationships(self, session, cls_, history_dataset_association, job): + obj = cls_(None, history_dataset_association) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.dataset.id == history_dataset_association.id + + +class TestJobToOutputDatasetCollectionAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_to_output_dataset_collection' + + def test_columns(self, session, cls_, history_dataset_collection_association, job): + name = 'a' + obj = cls_(name, history_dataset_collection_association) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.name == name + + def test_relationships(self, session, cls_, history_dataset_collection_association, job): + obj = cls_(None, history_dataset_collection_association) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.dataset_collection_instance.id == history_dataset_collection_association.id class TestLibraryDataset(BaseTest): - def test_table(self, cls_, model): + def test_table(self, cls_): assert cls_.__tablename__ == 'library_dataset' - def test_columns( - self, cls_, model, session, library_dataset_dataset_association, library_folder): - with dbcleanup(session, cls_): - folder = library_folder - order_id = 9 - create_time = datetime.now() - update_time = create_time + timedelta(hours=1) - name = 'a' - info = 'b' - deleted = False - purged = False + def test_columns(self, session, cls_, library_dataset_dataset_association, library_folder): + order_id = 9 + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + name = 'a' + info = 'b' + deleted = False + purged = False - obj = cls_() - obj.folder = folder - obj.order_id = order_id - obj.create_time = create_time - obj.update_time = update_time - obj.name = name - obj.info = info - obj.deleted = deleted - obj.purged = purged - obj_id = persist(session, obj) + obj = cls_() + obj.folder = library_folder + obj.order_id = order_id + obj.create_time = create_time + obj.update_time = update_time + obj.name = name + obj.info = info + obj.deleted = deleted + obj.purged = purged + with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert stored_obj.folder_id == folder.id + assert stored_obj.folder_id == library_folder.id assert stored_obj.order_id == order_id assert stored_obj.create_time == create_time assert stored_obj.update_time == update_time @@ -938,257 +1165,308 @@ class TestLibraryDataset(BaseTest): assert stored_obj.deleted == deleted assert stored_obj.purged == purged - def test_relationships(self, cls_, model, session, library_dataset_dataset_association, - library_folder, library_dataset_permission): - with dbcleanup(session, cls_): - folder = library_folder - obj = cls_() - obj.library_dataset_dataset_association = library_dataset_dataset_association - obj.folder = folder + def test_relationships( + self, + session, + cls_, + library_dataset_dataset_association, + library_folder, + library_dataset_permission, + model, + ): + obj = cls_() + obj.folder = library_folder + obj.library_dataset_dataset_association = library_dataset_dataset_association + obj.actions.append(library_dataset_permission) - ldda = model.LibraryDatasetDatasetAssociation() - ldda.library_dataset = obj - persist(session, ldda) - - obj.actions.append(library_dataset_permission) - obj_id = persist(session, obj) + ldda = model.LibraryDatasetDatasetAssociation() + ldda.library_dataset = obj + obj.actions.append(library_dataset_permission) + with dbcleanup(session, obj) as obj_id, dbcleanup(session, ldda): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id - assert stored_obj.folder.id == folder.id + assert stored_obj.folder.id == library_folder.id assert stored_obj.expired_datasets[0].id == ldda.id assert stored_obj.actions == [library_dataset_permission] -def test_JobToOutputLibraryDatasetAssociation(model, session, library_dataset_dataset_association, job): - cls = model.JobToOutputLibraryDatasetAssociation - assert cls.__tablename__ == 'job_to_output_library_dataset' +class TestJobToOutputLibraryDatasetAssociation(BaseTest): - with dbcleanup(session, cls): + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_to_output_library_dataset' + + def test_columns(self, session, cls_, library_dataset_dataset_association, job): name = 'a' - obj = cls(name, library_dataset_dataset_association) + obj = cls_(name, library_dataset_dataset_association) obj.job = job - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.ldda_id == library_dataset_dataset_association.id - assert stored_obj.name == name - # test mapped relationships - assert stored_obj.job.id == job.id - assert stored_obj.dataset.id == library_dataset_dataset_association.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.ldda_id == library_dataset_dataset_association.id + assert stored_obj.name == name + + def test_relationships(self, session, cls_, library_dataset_dataset_association, job): + obj = cls_(None, library_dataset_dataset_association) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.dataset.id == library_dataset_dataset_association.id -def test_Library(model, session, library_folder, library_permission): - cls = model.Library - assert cls.__tablename__ == 'library' - with dbcleanup(session, cls): +class TestLibrary(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library' + + def test_columns(self, session, cls_, library_folder): name, deleted, purged, description, synopsis = 'a', True, True, 'b', 'c' - obj = cls(name, description, synopsis, library_folder) create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(name, description, synopsis, library_folder) obj.create_time = create_time obj.update_time = update_time obj.deleted = deleted obj.purged = purged + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.root_folder_id == library_folder.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + assert stored_obj.description == description + assert stored_obj.synopsis == synopsis + + def test_relationships(self, session, cls_, library_folder, library_permission): + obj = cls_(None, None, None, library_folder) obj.actions.append(library_permission) - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.root_folder_id == library_folder.id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.name == name - assert stored_obj.deleted == deleted - assert stored_obj.purged == purged - assert stored_obj.description == description - assert stored_obj.synopsis == synopsis - # test mapped relationships - assert stored_obj.root_folder.id == library_folder.id - assert stored_obj.actions == [library_permission] + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.root_folder.id == library_folder.id + assert stored_obj.actions == [library_permission] -def test_LibraryDatasetCollectionAnnotationAssociation( - model, session, library_dataset_collection_association, user): - cls = model.LibraryDatasetCollectionAnnotationAssociation - assert cls.__tablename__ == 'library_dataset_collection_annotation_association' - with dbcleanup(session, cls): +class TestLibraryDatasetCollectionAnnotationAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset_collection_annotation_association' + + def test_columns(self, session, cls_, library_dataset_collection_association, user): annotation = 'a' - obj = cls() + obj = cls_() obj.user = user obj.dataset_collection = library_dataset_collection_association obj.annotation = annotation - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id - assert stored_obj.user_id == user.id - assert stored_obj.annotation == annotation - # test mapped relationships - assert stored_obj.dataset_collection.id == library_dataset_collection_association.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id + assert stored_obj.user_id == user.id + assert stored_obj.annotation == annotation + + def test_relationships(self, session, cls_, library_dataset_collection_association, user): + obj = cls_() + obj.user = user + obj.dataset_collection = library_dataset_collection_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset_collection.id == library_dataset_collection_association.id + assert stored_obj.user.id == user.id -def test_LibraryDatasetCollectionTagAssociation(model, session, library_dataset_collection_association, tag, user): - cls = model.LibraryDatasetCollectionTagAssociation - assert cls.__tablename__ == 'library_dataset_collection_tag_association' - with dbcleanup(session, cls): +class TestLibraryDatasetCollectionTagAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset_collection_tag_association' + + def test_columns(self, session, cls_, library_dataset_collection_association, tag, user): user_tname, value, user_value = 'a', 'b', 'c' - obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) obj.user_value = user_value obj.dataset_collection = library_dataset_collection_association - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - # test mapped relationships - assert stored_obj.dataset_collection.id == library_dataset_collection_association.id - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, library_dataset_collection_association, tag, user): + obj = cls_(user=user, tag=tag) + obj.dataset_collection = library_dataset_collection_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset_collection.id == library_dataset_collection_association.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id -def test_LibraryDatasetCollectionRatingAssociation( - model, session, library_dataset_collection_association, user): - cls = model.LibraryDatasetCollectionRatingAssociation - assert cls.__tablename__ == 'library_dataset_collection_rating_association' - with dbcleanup(session, cls): +class TestLibraryDatasetCollectionRatingAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset_collection_rating_association' + + def test_columns(self, session, cls_, library_dataset_collection_association, user): rating = 9 - obj = cls(user, library_dataset_collection_association, rating) - obj_id = persist(session, obj) + obj = cls_(user, library_dataset_collection_association, rating) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id - assert stored_obj.user_id == user.id - assert stored_obj.rating == rating - # test mapped relationships - assert stored_obj.dataset_collection.id == library_dataset_collection_association.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id + assert stored_obj.user_id == user.id + assert stored_obj.rating == rating + + def test_relationships(self, session, cls_, library_dataset_collection_association, user): + obj = cls_(user, library_dataset_collection_association, 1) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset_collection.id == library_dataset_collection_association.id + assert stored_obj.user.id == user.id -def test_LibraryDatasetDatasetAssociationPermissions( - model, session, library_dataset_dataset_association, role): - cls = model.LibraryDatasetDatasetAssociationPermissions - assert cls.__tablename__ == 'library_dataset_dataset_association_permissions' - with dbcleanup(session, cls): +class TestLibraryDatasetDatasetAssociationPermissions(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset_dataset_association_permissions' + + def test_columns(self, session, cls_, library_dataset_dataset_association, role): action = 'a' - obj = cls(action, library_dataset_dataset_association, role) create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(action, library_dataset_dataset_association, role) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.action == action - assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id - assert stored_obj.role_id == role.id - # test mapped relationships - assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id - assert stored_obj.role.id == role.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.action == action + assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert stored_obj.role_id == role.id + + def test_relationships(self, session, cls_, library_dataset_dataset_association, role): + obj = cls_(None, library_dataset_dataset_association, role) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert stored_obj.role.id == role.id -def test_LibraryDatasetDatasetAssociationTagAssociation( - model, session, library_dataset_dataset_association, tag, user): - cls = model.LibraryDatasetDatasetAssociationTagAssociation - assert cls.__tablename__ == 'library_dataset_dataset_association_tag_association' - with dbcleanup(session, cls): +class TestLibraryDatasetDatasetAssociationTagAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset_dataset_association_tag_association' + + def test_columns(self, session, cls_, library_dataset_dataset_association, tag, user): user_tname, value, user_value = 'a', 'b', 'c' - obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) obj.user_value = user_value obj.library_dataset_dataset_association = library_dataset_dataset_association - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - # test mapped relationships - assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, library_dataset_dataset_association, tag, user): + obj = cls_(user=user, tag=tag) + obj.library_dataset_dataset_association = library_dataset_dataset_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id -def test_LibraryDatasetPermissions(model, session, library_dataset, role): - cls = model.LibraryDatasetPermissions - assert cls.__tablename__ == 'library_dataset_permissions' - with dbcleanup(session, cls): +class TestLibraryDatasetPermissions(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset_permissions' + + def test_columns(self, session, cls_, library_dataset, role): action = 'a' - obj = cls(action, library_dataset, role) create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(action, library_dataset, role) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.action == action - assert stored_obj.library_dataset_id == library_dataset.id - assert stored_obj.role_id == role.id - # test mapped relationships - assert stored_obj.library_dataset.id == library_dataset.id - assert stored_obj.role.id == role.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.action == action + assert stored_obj.library_dataset_id == library_dataset.id + assert stored_obj.role_id == role.id + + def test_relationships(self, session, cls_, library_dataset, role): + obj = cls_(None, library_dataset, role) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library_dataset.id == library_dataset.id + assert stored_obj.role.id == role.id class TestLibraryFolder(BaseTest): - def test_table(self, cls_, model): + def test_table(self, cls_): assert cls_.__tablename__ == 'library_folder' assert has_index(cls_.__table__, ('name',)) - def test_columns(self, cls_, model, session, library_folder): - with dbcleanup(session, cls_): - parent = library_folder - create_time = datetime.now() - update_time = create_time + timedelta(hours=1) - name = 'a' - description = 'b' - order_id = 9 - item_count = 1 - deleted = False - purged = False - genome_build = 'c' + def test_columns(self, session, cls_, library_folder): + parent = library_folder + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + name = 'a' + description = 'b' + order_id = 9 + item_count = 1 + deleted = False + purged = False + genome_build = 'c' - obj = cls_() - obj.parent = parent - obj.create_time = create_time - obj.update_time = update_time - obj.name = name - obj.description = description - obj.order_id = order_id - obj.item_count = item_count - obj.deleted = deleted - obj.purged = purged - obj.genome_build = genome_build - obj_id = persist(session, obj) + obj = cls_() + obj.parent = parent + obj.create_time = create_time + obj.update_time = update_time + obj.name = name + obj.description = description + obj.order_id = order_id + obj.item_count = item_count + obj.deleted = deleted + obj.purged = purged + obj.genome_build = genome_build + with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id assert stored_obj.parent_id == parent.id @@ -1203,22 +1481,19 @@ class TestLibraryFolder(BaseTest): assert stored_obj.genome_build == genome_build def test_relationships( - self, cls_, model, session, library_folder, library_dataset, library, library_folder_permission): - with dbcleanup(session, cls_): - folder1 = model.LibraryFolder() + self, session, cls_, model, library_folder, library_dataset, library, library_folder_permission): + obj = cls_() + obj.parent = library_folder + folder1 = model.LibraryFolder() + obj.folders.append(folder1) + obj.library_root.append(library) + obj.actions.append(library_folder_permission) - obj = cls_() - # no back reference, so dataset does not update folder; so we have to flush to the db - # todo: ..but why is there no back reference? - library_dataset.folder = obj - persist(session, library_dataset) - - obj.parent = library_folder - obj.folders.append(folder1) - obj.library_root.append(library) - obj.actions.append(library_folder_permission) - obj_id = persist(session, obj) + # There's no back reference, so dataset does not update folder; so we have to flush to the database + # via dbcleanup() context manager. TODO: ..but why is there no back reference? + library_dataset.folder = obj + with dbcleanup(session, obj) as obj_id, dbcleanup(session, library_dataset): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == library_folder.id assert stored_obj.folders == [folder1] @@ -1230,74 +1505,80 @@ class TestLibraryFolder(BaseTest): assert stored_obj.active_datasets[0].id == library_dataset.id -def test_LibraryFolderPermissions(model, session, library_folder, role): - cls = model.LibraryFolderPermissions - assert cls.__tablename__ == 'library_folder_permissions' - with dbcleanup(session, cls): +class TestLibraryFolderPermissions(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_folder_permissions' + + def test_columns(self, session, cls_, library_folder, role): action = 'a' - obj = cls(action, library_folder, role) + obj = cls_(action, library_folder, role) create_time = datetime.now() update_time = create_time + timedelta(hours=1) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.action == action - assert stored_obj.library_folder_id == library_folder.id - assert stored_obj.role_id == role.id - # test mapped relationships - assert stored_obj.folder.id == library_folder.id - assert stored_obj.role.id == role.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.action == action + assert stored_obj.library_folder_id == library_folder.id + assert stored_obj.role_id == role.id + + def test_relationships(self, session, cls_, library_folder, role): + obj = cls_(None, library_folder, role) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.folder.id == library_folder.id + assert stored_obj.role.id == role.id -def test_LibraryPermissions(model, session, library, role): - cls = model.LibraryPermissions - assert cls.__tablename__ == 'library_permissions' - with dbcleanup(session, cls): +class TestLibraryPermissions(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_permissions' + + def test_columns(self, session, cls_, library, role): action = 'a' - obj = cls(action, library, role) + obj = cls_(action, library, role) create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(action, library, role) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.action == action - assert stored_obj.library_id == library.id - assert stored_obj.role_id == role.id - # test mapped relationships - assert stored_obj.library.id == library.id - assert stored_obj.role.id == role.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.action == action + assert stored_obj.library_id == library.id + assert stored_obj.role_id == role.id + + def test_relationships(self, session, cls_, library, role): + obj = cls_(None, library, role) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library.id == library.id + assert stored_obj.role.id == role.id -def test_Page( - model, - session, - user, - page_revision, - page_tag_association, - page_annotation_association, - page_rating_association, - page_user_share_association, -): - cls = model.Page - assert cls.__tablename__ == 'page' - assert has_index(cls.__table__, ('slug',)) - with dbcleanup(session, cls): +class TestPage(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'page' + assert has_index(cls_.__table__, ('slug',)) + + def test_columns(self, session, cls_, user, page_revision): title, deleted, importable, slug, published = 'a', True, True, 'b', True create_time = datetime.now() update_time = create_time + timedelta(hours=1) - obj = cls() + obj = cls_() obj.user = user obj.title = title obj.deleted = deleted @@ -1308,411 +1589,522 @@ def test_Page( obj.update_time = update_time # This is OK for this test; however, page_revision.page != obj. Can we do better? obj.latest_revision = page_revision + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.user_id == user.id + assert stored_obj.latest_revision_id == page_revision.id + assert stored_obj.title == title + assert stored_obj.deleted == deleted + assert stored_obj.importable == importable + assert stored_obj.slug == slug + assert stored_obj.published == published + + def test_relationships( + self, + session, + cls_, + user, + page_revision, + page_tag_association, + page_annotation_association, + page_rating_association, + page_user_share_association, + ): + obj = cls_() + obj.user = user + # This is OK for this test; however, page_revision.page != obj. Can we do better? + obj.latest_revision = page_revision obj.revisions.append(page_revision) obj.tags.append(page_tag_association) obj.annotations.append(page_annotation_association) obj.ratings.append(page_rating_association) obj.users_shared_with.append(page_user_share_association) - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.user_id == user.id - assert stored_obj.latest_revision_id == page_revision.id - assert stored_obj.title == title - assert stored_obj.deleted == deleted - assert stored_obj.importable == importable - assert stored_obj.slug == slug - assert stored_obj.published == published - # This doesn't test the average amount, just the mapping. - assert stored_obj.average_rating == page_rating_association.rating - # test mapped relationships - assert stored_obj.user.id == user.id - assert stored_obj.revisions == [page_revision] - assert stored_obj.latest_revision.id == page_revision.id - assert stored_obj.tags == [page_tag_association] - assert stored_obj.annotations == [page_annotation_association] - assert stored_obj.ratings == [page_rating_association] - assert stored_obj.users_shared_with == [page_user_share_association] + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id + assert stored_obj.revisions == [page_revision] + assert stored_obj.latest_revision.id == page_revision.id + assert stored_obj.tags == [page_tag_association] + assert stored_obj.annotations == [page_annotation_association] + assert stored_obj.ratings == [page_rating_association] + assert stored_obj.users_shared_with == [page_user_share_association] + # This doesn't test the average amount, just the mapping. + assert stored_obj.average_rating == page_rating_association.rating - -def test_Page_average_rating(model, session, page, user): - cls = model.PageRatingAssociation - with dbcleanup(session, cls): + def test_average_rating(self, model, session, page, user): # Page has been expunged; to access its deferred properties, # it needs to be added back to the session. session.add(page) assert page.average_rating is None # With no ratings, we expect None. # Create ratings + to_cleanup = [] for rating in (1, 2, 3, 4, 5): - r = cls(user, page) - r.rating = rating - persist(session, r) + page_rating_assoc = model.PageRatingAssociation(user, page) + page_rating_assoc.rating = rating + persist(session, page_rating_assoc) + to_cleanup.append(page_rating_assoc) assert page.average_rating == 3.0 # Expect average after ratings added. + # Cleanup: remove ratings from database + delete_from_database(session, to_cleanup) -def test_PageAnnotationAssociation(model, session, page, user): - cls = model.PageAnnotationAssociation - assert cls.__tablename__ == 'page_annotation_association' - assert has_index(cls.__table__, ('annotation',)) - with dbcleanup(session, cls): +class TestPageAnnotationAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'page_annotation_association' + assert has_index(cls_.__table__, ('annotation',)) + + def test_columns(self, session, cls_, page, user): annotation = 'a' - obj = cls() + obj = cls_() obj.user = user obj.page = page obj.annotation = annotation - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.page_id == page.id - assert stored_obj.user_id == user.id - assert stored_obj.annotation == annotation - # test mapped relationships - assert stored_obj.page.id == page.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.page_id == page.id + assert stored_obj.user_id == user.id + assert stored_obj.annotation == annotation - -def test_PageRatingAssociation(model, session, page, user): - cls = model.PageRatingAssociation - assert cls.__tablename__ == 'page_rating_association' - with dbcleanup(session, cls): - rating = 9 - obj = cls(user, page, rating) - obj_id = persist(session, obj) - - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.page_id == page.id - assert stored_obj.user_id == user.id - assert stored_obj.rating == rating - # test mapped relationships - assert stored_obj.page.id == page.id - assert stored_obj.user.id == user.id - - -def test_PageRevision(model, session, page): - cls = model.PageRevision - assert cls.__tablename__ == 'page_revision' - with dbcleanup(session, cls): - obj = cls() + def test_relationships(self, session, cls_, page, user): + obj = cls_() + obj.user = user obj.page = page + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.page.id == page.id + assert stored_obj.user.id == user.id + + +class TestPageRatingAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'page_rating_association' + + def test_columns(self, session, cls_, page, user): + rating = 9 + obj = cls_(user, page, rating) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.page_id == page.id + assert stored_obj.user_id == user.id + assert stored_obj.rating == rating + + def test_relationships(self, session, cls_, page, user): + obj = cls_(user, page, 1) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.page.id == page.id + assert stored_obj.user.id == user.id + + +class TestPageRevision(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'page_revision' + + def test_columns(self, session, cls_, model, page): + title, content = 'a', 'b' create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_() + obj.page = page obj.create_time = create_time obj.update_time = update_time - title, content = 'a', 'b' obj.title = title obj.content = content - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.page_id == page.id - assert stored_obj.title == title - assert stored_obj.content == content - assert stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT - # test mapped relationships - assert stored_obj.page.id == page.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.page_id == page.id + assert stored_obj.title == title + assert stored_obj.content == content + assert stored_obj.content_format == model.PageRevision.DEFAULT_CONTENT_FORMAT + + def test_relationships(self, session, cls_, page): + obj = cls_() + obj.page = page + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.page.id == page.id -def test_PageTagAssociation(model, session, page, tag, user): - cls = model.PageTagAssociation - assert cls.__tablename__ == 'page_tag_association' - with dbcleanup(session, cls): +class TestPageTagAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'page_tag_association' + + def test_columns(self, session, cls_, page, tag, user): user_tname, value, user_value = 'a', 'b', 'c' - obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) obj.user_value = user_value obj.page = page - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.page_id == page.id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - # test mapped relationships - assert stored_obj.page.id == page.id - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.page_id == page.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, page, tag, user): + obj = cls_(user=user, tag=tag) + obj.page = page + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.page.id == page.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id -def test_PageUserShareAssociation(model, session, page, user): - cls = model.PageUserShareAssociation - assert cls.__tablename__ == 'page_user_share_association' - with dbcleanup(session, cls): - obj = cls() +class TestPageUserShareAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'page_user_share_association' + + def test_columns(self, session, cls_, page, user): + obj = cls_() obj.page = page obj.user = user - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.page_id == page.id - assert stored_obj.user_id == user.id - # test mapped relationships - assert stored_obj.page.id == page.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.page_id == page.id + assert stored_obj.user_id == user.id + + def test_relationships(self, session, cls_, page, user): + obj = cls_() + obj.page = page + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.page.id == page.id + assert stored_obj.user.id == user.id -def test_PasswordResetToken(model, session, user): - cls = model.PasswordResetToken - assert cls.__tablename__ == 'password_reset_token' - with dbcleanup(session, cls): +class TestPasswordResetToken(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'password_reset_token' + + def test_columns_and_relationships(self, session, cls_, user): token = get_random_string() expiration_time = datetime.now() - obj = cls(user, token) + obj = cls_(user, token) obj.expiration_time = expiration_time - persist(session, obj, return_id=False) - stmt = select(cls).where(cls.token == token) - stored_obj = get_stored_obj(session, cls, stmt=stmt) - # test mapped columns - assert stored_obj.token == token - assert stored_obj.expiration_time == expiration_time - # test mapped relationships - assert stored_obj.user.id == user.id + where_clause = cls_.token == token + + with dbcleanup(session, obj, where_clause): + stored_obj = get_stored_obj2(session, cls_, where_clause=where_clause) + # test columns + assert stored_obj.token == token + assert stored_obj.expiration_time == expiration_time + assert stored_obj.user_id == user.id + # test relationships + assert stored_obj.user.id == user.id -def test_PSAAssociation(model, session): - cls = model.PSAAssociation - assert cls.__tablename__ == 'psa_association' - with dbcleanup(session, cls): +class TestPSAAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'psa_association' + + def test_columns(self, session, cls_): server_url, handle, secret, issued, lifetime, assoc_type = 'a', 'b', 'c', 1, 2, 'd' - obj = cls(server_url, handle, secret, issued, lifetime, assoc_type) - obj_id = persist(session, obj) + obj = cls_(server_url, handle, secret, issued, lifetime, assoc_type) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.server_url == server_url - assert stored_obj.handle == handle - assert stored_obj.secret == secret - assert stored_obj.issued == issued - assert stored_obj.lifetime == lifetime - assert stored_obj.assoc_type == assoc_type + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.server_url == server_url + assert stored_obj.handle == handle + assert stored_obj.secret == secret + assert stored_obj.issued == issued + assert stored_obj.lifetime == lifetime + assert stored_obj.assoc_type == assoc_type -def test_PSACode(model, session): - cls = model.PSACode - assert cls.__tablename__ == 'psa_code' - assert has_unique_constraint(cls.__table__, ('code', 'email')) - with dbcleanup(session, cls): +class TestPSACode(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'psa_code' + assert has_unique_constraint(cls_.__table__, ('code', 'email')) + + def test_columns(self, session, cls_): email, code = 'a', get_random_string() - obj = cls(email, code) - obj_id = persist(session, obj) + obj = cls_(email, code) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.email == email - assert stored_obj.code == code + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.email == email + assert stored_obj.code == code -def test_PSANonce(model, session): - cls = model.PSANonce - assert cls.__tablename__ == 'psa_nonce' - with dbcleanup(session, cls): +class TestPSANonce(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'psa_nonce' + + def test_columns(self, session, cls_): server_url, timestamp, salt = 'a', 1, 'b' - obj = cls(server_url, timestamp, salt) - obj_id = persist(session, obj) + obj = cls_(server_url, timestamp, salt) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.server_url - assert stored_obj.timestamp == timestamp - assert stored_obj.salt == salt + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.server_url + assert stored_obj.timestamp == timestamp + assert stored_obj.salt == salt -def test_PSAPartial(model, session): - cls = model.PSAPartial - assert cls.__tablename__ == 'psa_partial' - with dbcleanup(session, cls): +class TestPSAPartial(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'psa_partial' + + def test_columns(self, session, cls_): token, data, next_step, backend = 'a', 'b', 1, 'c' - obj = cls(token, data, next_step, backend) - obj_id = persist(session, obj) + obj = cls_(token, data, next_step, backend) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.token == token - assert stored_obj.data == data - assert stored_obj.next_step == next_step - assert stored_obj.backend == backend + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.token == token + assert stored_obj.data == data + assert stored_obj.next_step == next_step + assert stored_obj.backend == backend -def test_Quota( - model, - session, - default_quota_association, - group_quota_association, - user_quota_association -): - cls = model.Quota - assert cls.__tablename__ == 'quota' - with dbcleanup(session, cls): +class TestQuota(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'quota' + + def test_columns(self, session, cls_): name, description, amount, operation = get_random_string(), 'b', 42, '+' - obj = cls(name, description, amount, operation) create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(name, description, amount, operation) obj.create_time = create_time obj.update_time = update_time + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.description == description + assert stored_obj.bytes == amount + assert stored_obj.operation == operation + assert stored_obj.deleted is False + + def test_relationships(self, session, cls_, default_quota_association, group_quota_association, user_quota_association): + def add_association(assoc_object, assoc_attribute): assoc_object.quota = obj getattr(obj, assoc_attribute).append(assoc_object) + obj = cls_(None, None, 1, '+') add_association(default_quota_association, 'default') add_association(group_quota_association, 'groups') add_association(user_quota_association, 'users') - obj_id = persist(session, obj) - - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.name == name - assert stored_obj.description == description - assert stored_obj.bytes == amount - assert stored_obj.operation == operation - assert stored_obj.deleted is False - # test mapped relationships - assert stored_obj.default == [default_quota_association] - assert stored_obj.groups == [group_quota_association] - assert stored_obj.users == [user_quota_association] + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.default == [default_quota_association] + assert stored_obj.groups == [group_quota_association] + assert stored_obj.users == [user_quota_association] -def test_Role( - model, +class TestRole(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'role' + + def test_columns(self, session, cls_): + name, description, type_, deleted = get_random_string(), 'b', cls_.types.SYSTEM, True + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj = cls_(name, description, type_, deleted) + obj.create_time = create_time + obj.update_time = update_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.description == description + assert stored_obj.type == type_ + assert stored_obj.deleted == deleted + + def test_relationships( + self, session, + cls_, dataset_permission, group_role_association, library_permission, library_folder_permission, library_dataset_permission, library_dataset_dataset_association_permission, -): - cls = model.Role - assert cls.__tablename__ == 'role' - with dbcleanup(session, cls): - name, description, type_, deleted = get_random_string(), 'b', cls.types.SYSTEM, True - obj = cls(name, description, type_, deleted) - create_time = datetime.now() - update_time = create_time + timedelta(hours=1) - obj.create_time = create_time - obj.update_time = update_time + ): + name, description, type_ = get_random_string(), 'b', cls_.types.SYSTEM + obj = cls_(name, description, type_) obj.dataset_actions.append(dataset_permission) obj.library_actions.append(library_permission) obj.library_folder_actions.append(library_folder_permission) obj.library_dataset_actions.append(library_dataset_permission) obj.library_dataset_dataset_actions.append(library_dataset_dataset_association_permission) obj.groups.append(group_role_association) - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.name == name - assert stored_obj.description == description - assert stored_obj.type == type_ - assert stored_obj.deleted == deleted - # test mapped relationships - assert stored_obj.dataset_actions == [dataset_permission] - assert stored_obj.groups == [group_role_association] - assert stored_obj.library_actions == [library_permission] - assert stored_obj.library_folder_actions == [library_folder_permission] - assert stored_obj.library_dataset_actions == [library_dataset_permission] - assert stored_obj.library_dataset_dataset_actions == [library_dataset_dataset_association_permission] + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset_actions == [dataset_permission] + assert stored_obj.groups == [group_role_association] + assert stored_obj.library_actions == [library_permission] + assert stored_obj.library_folder_actions == [library_folder_permission] + assert stored_obj.library_dataset_actions == [library_dataset_permission] + assert stored_obj.library_dataset_dataset_actions == [library_dataset_dataset_association_permission] -def test_StoredWorkflowAnnotationAssociation(model, session, stored_workflow, user): - cls = model.StoredWorkflowAnnotationAssociation - assert cls.__tablename__ == 'stored_workflow_annotation_association' - assert has_index(cls.__table__, ('annotation',)) - with dbcleanup(session, cls): +class TestStoredWorkflowAnnotationAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'stored_workflow_annotation_association' + assert has_index(cls_.__table__, ('annotation',)) + + def test_columns(self, session, cls_, stored_workflow, user): annotation = 'a' - obj = cls() + obj = cls_() obj.stored_workflow = stored_workflow obj.user = user obj.annotation = annotation - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.stored_workflow_id == stored_workflow.id - assert stored_obj.user_id == user.id - assert stored_obj.annotation == annotation - # test mapped relationships - assert stored_obj.stored_workflow.id == stored_workflow.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.stored_workflow_id == stored_workflow.id + assert stored_obj.user_id == user.id + assert stored_obj.annotation == annotation + + def test_relationships(self, session, cls_, stored_workflow, user): + obj = cls_() + obj.stored_workflow = stored_workflow + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.stored_workflow.id == stored_workflow.id + assert stored_obj.user.id == user.id -def test_StoredWorkflowRatingAssociation(model, session, stored_workflow, user): - cls = model.StoredWorkflowRatingAssociation - assert cls.__tablename__ == 'stored_workflow_rating_association' - with dbcleanup(session, cls): +class TestStoredWorkflowRatingAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'stored_workflow_rating_association' + + def test_columns(self, session, cls_, stored_workflow, user): rating = 9 - obj = cls(user, stored_workflow, rating) - obj_id = persist(session, obj) + obj = cls_(user, stored_workflow, rating) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.stored_workflow_id == stored_workflow.id - assert stored_obj.user_id == user.id - assert stored_obj.rating == rating - # test mapped relationships - assert stored_obj.stored_workflow.id == stored_workflow.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.stored_workflow_id == stored_workflow.id + assert stored_obj.user_id == user.id + assert stored_obj.rating == rating + + def test_relationships(self, session, cls_, stored_workflow, user): + obj = cls_(user, stored_workflow, 1) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.stored_workflow.id == stored_workflow.id + assert stored_obj.user.id == user.id -def test_StoredWorkflowTagAssociation(model, session, stored_workflow, tag, user): - cls = model.StoredWorkflowTagAssociation - assert cls.__tablename__ == 'stored_workflow_tag_association' - with dbcleanup(session, cls): +class TestStoredWorkflowTagAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'stored_workflow_tag_association' + + def test_columns(self, session, cls_, stored_workflow, tag, user): user_tname, value, user_value = 'a', 'b', 'c' - obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) obj.user_value = user_value obj.stored_workflow = stored_workflow - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.stored_workflow_id == stored_workflow.id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - # test mapped relationships - assert stored_obj.stored_workflow.id == stored_workflow.id - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.stored_workflow_id == stored_workflow.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, stored_workflow, tag, user): + obj = cls_(user=user, tag=tag) + obj.stored_workflow = stored_workflow + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.stored_workflow.id == stored_workflow.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id -def test_Tag( - model, +class TestTag(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'tag' + assert has_unique_constraint(cls_.__table__, ('name',)) + + def test_columns(self, session, cls_): + parent_tag = cls_() + type_, name = 1, 'a' + obj = cls_(type=type_, name=name) + obj.parent = parent_tag + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.type == type_ + assert stored_obj.parent_id == parent_tag.id + assert stored_obj.name == name + + def test_relationships( + self, session, + cls_, history_tag_association, history_dataset_association_tag_association, library_dataset_dataset_association_tag_association, @@ -1723,16 +2115,10 @@ def test_Tag( history_dataset_collection_tag_association, library_dataset_collection_tag_association, tool_tag_association, -): - cls = model.Tag - assert cls.__tablename__ == 'tag' - assert has_unique_constraint(cls.__table__, ('name',)) - - with dbcleanup(session, cls): - parent_tag = cls() - child_tag = cls() - type_, name = 1, 'a' - obj = cls(type=type_, name=name) + ): + obj = cls_() + parent_tag = cls_() + child_tag = cls_() obj.parent = parent_tag obj.children.append(child_tag) @@ -1751,33 +2137,28 @@ def test_Tag( add_association(library_dataset_collection_tag_association, 'tagged_library_dataset_collections') add_association(tool_tag_association, 'tagged_tools') - obj_id = persist(session, obj) - - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.type == type_ - assert stored_obj.parent_id == parent_tag.id - assert stored_obj.name == name - # test mapped relationships - assert stored_obj.parent.id == parent_tag.id - assert stored_obj.children == [child_tag] - assert stored_obj.tagged_histories == [history_tag_association] - assert stored_obj.tagged_history_dataset_associations == [history_dataset_association_tag_association] - assert stored_obj.tagged_library_dataset_dataset_associations == [library_dataset_dataset_association_tag_association] - assert stored_obj.tagged_pages == [page_tag_association] - assert stored_obj.tagged_workflow_steps == [workflow_step_tag_association] - assert stored_obj.tagged_stored_workflows == [stored_workflow_tag_association] - assert stored_obj.tagged_visualizations == [visualization_tag_association] - assert stored_obj.tagged_history_dataset_collections == [history_dataset_collection_tag_association] - assert stored_obj.tagged_library_dataset_collections == [library_dataset_collection_tag_association] - assert stored_obj.tagged_tools == [tool_tag_association] + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.parent.id == parent_tag.id + assert stored_obj.children == [child_tag] + assert stored_obj.tagged_histories == [history_tag_association] + assert stored_obj.tagged_history_dataset_associations == [history_dataset_association_tag_association] + assert stored_obj.tagged_library_dataset_dataset_associations == [library_dataset_dataset_association_tag_association] + assert stored_obj.tagged_pages == [page_tag_association] + assert stored_obj.tagged_workflow_steps == [workflow_step_tag_association] + assert stored_obj.tagged_stored_workflows == [stored_workflow_tag_association] + assert stored_obj.tagged_visualizations == [visualization_tag_association] + assert stored_obj.tagged_history_dataset_collections == [history_dataset_collection_tag_association] + assert stored_obj.tagged_library_dataset_collections == [library_dataset_collection_tag_association] + assert stored_obj.tagged_tools == [tool_tag_association] -def test_Task(model, session, job, task_metric_numeric, task_metric_text): - cls = model.Task - assert cls.__tablename__ == 'task' - with dbcleanup(session, cls): +class TestTask(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'task' + + def test_columns(self, session, cls_, model, job): create_time = datetime.now() execution_time = create_time + timedelta(hours=1) update_time = execution_time + timedelta(hours=1) @@ -1798,7 +2179,7 @@ def test_Task(model, session, job, task_metric_numeric, task_metric_text): task_runner_external_id = 'm' prepare_input_files_cmd = 'n' - obj = cls(job, working_directory, prepare_input_files_cmd) + obj = cls_(job, working_directory, prepare_input_files_cmd) obj.create_time = create_time obj.execution_time = execution_time obj.update_time = update_time @@ -1816,406 +2197,506 @@ def test_Task(model, session, job, task_metric_numeric, task_metric_text): obj.traceback = traceback obj.task_runner_name = task_runner_name obj.task_runner_external_id = task_runner_external_id + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.execution_time == execution_time + assert stored_obj.update_time == update_time + assert stored_obj.state == state + assert stored_obj.command_line == command_line + assert stored_obj.param_filename == param_filename + assert stored_obj.runner_name == runner_name + assert stored_obj.job_stdout == job_stdout + assert stored_obj.job_stderr == job_stderr + assert stored_obj.tool_stdout == tool_stdout + assert stored_obj.tool_stderr == tool_stderr + assert stored_obj.exit_code == exit_code + assert stored_obj.job_messages == job_messages + assert stored_obj.info == info + assert stored_obj.traceback == traceback + assert stored_obj.task_runner_name == task_runner_name + assert stored_obj.task_runner_external_id == task_runner_external_id + + def test_relationships(self, session, cls_, job, task_metric_numeric, task_metric_text): + obj = cls_(job, None, None) obj.numeric_metrics.append(task_metric_numeric) obj.text_metrics.append(task_metric_text) - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.execution_time == execution_time - assert stored_obj.update_time == update_time - assert stored_obj.state == state - assert stored_obj.command_line == command_line - assert stored_obj.param_filename == param_filename - assert stored_obj.runner_name == runner_name - assert stored_obj.job_stdout == job_stdout - assert stored_obj.job_stderr == job_stderr - assert stored_obj.tool_stdout == tool_stdout - assert stored_obj.tool_stderr == tool_stderr - assert stored_obj.exit_code == exit_code - assert stored_obj.job_messages == job_messages - assert stored_obj.info == info - assert stored_obj.traceback == traceback - assert stored_obj.task_runner_name == task_runner_name - assert stored_obj.task_runner_external_id == task_runner_external_id - # test mapped relationships - assert stored_obj.job.id == job.id - assert stored_obj.numeric_metrics == [task_metric_numeric] - assert stored_obj.text_metrics == [task_metric_text] + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.numeric_metrics == [task_metric_numeric] + assert stored_obj.text_metrics == [task_metric_text] -def test_TaskMetricNumeric(model, session, task): - cls = model.TaskMetricNumeric - assert cls.__tablename__ == 'task_metric_numeric' - with dbcleanup(session, cls): +class TestTaskMetricNumeric(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'task_metric_numeric' + + def test_columns(self, session, cls_, task): plugin, metric_name, metric_value = 'a', 'b', 9 - obj = cls(plugin, metric_name, metric_value) + obj = cls_(plugin, metric_name, metric_value) obj.task = task - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.task_id == task.id - assert stored_obj.plugin == plugin - assert stored_obj.metric_value == metric_value - # test mapped relationships - assert stored_obj.task.id == task.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.task_id == task.id + assert stored_obj.plugin == plugin + assert stored_obj.metric_value == metric_value + + def test_relationships(self, session, cls_, task): + obj = cls_(None, None, None) + obj.task = task + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.task.id == task.id -def test_TaskMetricText(model, session, task): - cls = model.TaskMetricText - assert cls.__tablename__ == 'task_metric_text' - with dbcleanup(session, cls): +class TestTaskMetricText(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'task_metric_text' + + def test_columns(self, session, cls_, task): plugin, metric_name, metric_value = 'a', 'b', 'c' - obj = cls(plugin, metric_name, metric_value) + obj = cls_(plugin, metric_name, metric_value) obj.task = task - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.task_id == task.id - assert stored_obj.plugin == plugin - assert stored_obj.metric_value == metric_value - # test mapped relationships - assert stored_obj.task.id == task.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.task_id == task.id + assert stored_obj.plugin == plugin + assert stored_obj.metric_value == metric_value + + def test_relationships(self, session, cls_, task): + obj = cls_(None, None, None) + obj.task = task + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.task.id == task.id -def test_ToolTagAssociation(model, session, tag, user): - cls = model.ToolTagAssociation - assert cls.__tablename__ == 'tool_tag_association' - with dbcleanup(session, cls): +class TestToolTagAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'tool_tag_association' + + def test_columns(self, session, cls_, tag, user): user_tname, value, user_value, tool_id = 'a', 'b', 'c', 'd' - obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) obj.user_value = user_value obj.tool_id = tool_id - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.tool_id == tool_id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - # test mapped relationships - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.tool_id == tool_id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, tag, user): + obj = cls_(user=user, tag=tag) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id -def test_UserAction(model, session, user, galaxy_session): - cls = model.UserAction - assert cls.__tablename__ == 'user_action' - with dbcleanup(session, cls): +class TestUserAction(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'user_action' + + def test_columns(self, session, cls_, user, galaxy_session): action, params, context = 'a', 'b', 'c' - obj = cls(user, galaxy_session.id, action, params, context) create_time = datetime.now() + obj = cls_(user, galaxy_session.id, action, params, context) obj.create_time = create_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.user_id == user.id - assert stored_obj.session_id == galaxy_session.id - assert stored_obj.action == action - assert stored_obj.context == context - assert stored_obj.params == params - # test mapped relationships - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.user_id == user.id + assert stored_obj.session_id == galaxy_session.id + assert stored_obj.action == action + assert stored_obj.context == context + assert stored_obj.params == params + + def test_relationships(self, session, cls_, user, galaxy_session): + obj = cls_(user, galaxy_session.id, None, None, None) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id -def test_UserAddress(model, session, user): - cls = model.UserAddress - assert cls.__tablename__ == 'user_address' - with dbcleanup(session, cls): +class TestUserAddress(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'user_address' + + def test_columns_and_relationships(self, session, cls_, user): desc, name, institution, address, city, state, postal_code, country, phone, deleted, purged = \ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', True, False - obj = cls(user, desc, name, institution, address, city, state, postal_code, country, phone) + obj = cls_(user, desc, name, institution, address, city, state, postal_code, country, phone) create_time = datetime.now() update_time = create_time + timedelta(hours=1) obj.create_time = create_time obj.update_time = update_time obj.deleted = deleted obj.purged = purged - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.user_id == user.id - assert stored_obj.desc == desc - assert stored_obj.name == name - assert stored_obj.institution == institution - assert stored_obj.address == address - assert stored_obj.city == city - assert stored_obj.state == state - assert stored_obj.postal_code == postal_code - assert stored_obj.country == country - assert stored_obj.phone == phone - assert stored_obj.deleted == deleted - assert stored_obj.purged == purged - # test mapped relationships - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + # test columns + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.user_id == user.id + assert stored_obj.desc == desc + assert stored_obj.name == name + assert stored_obj.institution == institution + assert stored_obj.address == address + assert stored_obj.city == city + assert stored_obj.state == state + assert stored_obj.postal_code == postal_code + assert stored_obj.country == country + assert stored_obj.phone == phone + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + # test relationships + assert stored_obj.user.id == user.id -def test_UserAuthnzToken(model, session, user, cloud_authz): - cls = model.UserAuthnzToken - assert cls.__tablename__ == 'oidc_user_authnz_tokens' - assert has_unique_constraint(cls.__table__, ('provider', 'uid')) - with dbcleanup(session, cls): +class TestUserAuthnzToken(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'oidc_user_authnz_tokens' + assert has_unique_constraint(cls_.__table__, ('provider', 'uid')) + + def test_columns(self, session, cls_, user): provider, uid, extra_data, lifetime, assoc_type = get_random_string(), 'b', 'c', 1, 'd' - obj = cls(provider, uid, extra_data, lifetime, assoc_type, user) + obj = cls_(provider, uid, extra_data, lifetime, assoc_type, user) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user_id == user.id + assert stored_obj.uid == uid + assert stored_obj.provider == provider + assert stored_obj.extra_data == extra_data + assert stored_obj.lifetime == lifetime + assert stored_obj.assoc_type == assoc_type + + def test_relationships(self, session, cls_, user, cloud_authz): + obj = cls_(get_random_string(), None, user=user) obj.cloudauthz.append(cloud_authz) - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.user_id == user.id - assert stored_obj.uid == uid - assert stored_obj.provider == provider - assert stored_obj.extra_data == extra_data - assert stored_obj.lifetime == lifetime - assert stored_obj.assoc_type == assoc_type - # test mapped relationships - assert stored_obj.cloudauthz == [cloud_authz] - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.cloudauthz == [cloud_authz] + assert stored_obj.user.id == user.id -def test_UserGroupAssociation(model, session, user, group): - cls = model.UserGroupAssociation - assert cls.__tablename__ == 'user_group_association' - with dbcleanup(session, cls): - obj = cls(user, group) +class TestUserGroupAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'user_group_association' + + def test_columns(self, session, cls_, user, group): create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(user, group) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.user_id == user.id - assert stored_obj.group_id == group.id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - # test mapped relationships - assert stored_obj.user.id == user.id - assert stored_obj.group.id == group.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user_id == user.id + assert stored_obj.group_id == group.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + + def test_relationships(self, session, cls_, user, group): + obj = cls_(user, group) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id + assert stored_obj.group.id == group.id -def test_UserQuotaAssociation(model, session, user, quota): - cls = model.UserQuotaAssociation - assert cls.__tablename__ == 'user_quota_association' - with dbcleanup(session, cls): - obj = cls(user, quota) +class TestUserQuotaAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'user_quota_association' + + def test_columns(self, session, cls_, user, quota): create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(user, quota) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.user_id == user.id - assert stored_obj.quota_id == quota.id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - # test mapped relationships - assert stored_obj.user.id == user.id - assert stored_obj.quota.id == quota.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user_id == user.id + assert stored_obj.quota_id == quota.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + + def test_relationships(self, session, cls_, user, quota): + obj = cls_(user, quota) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id + assert stored_obj.quota.id == quota.id -def test_VisualizationAnnotationAssociation(model, session, visualization, user): - cls = model.VisualizationAnnotationAssociation - assert cls.__tablename__ == 'visualization_annotation_association' - assert has_index(cls.__table__, ('annotation',)) - with dbcleanup(session, cls): +class TestVisualizationAnnotationAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'visualization_annotation_association' + assert has_index(cls_.__table__, ('annotation',)) + + def test_columns(self, session, cls_, visualization, user): annotation = 'a' - obj = cls() + obj = cls_() obj.user = user obj.visualization = visualization obj.annotation = annotation - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.visualization_id == visualization.id - assert stored_obj.user_id == user.id - assert stored_obj.annotation == annotation - # test mapped relationships - assert stored_obj.visualization.id == visualization.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.visualization_id == visualization.id + assert stored_obj.user_id == user.id + assert stored_obj.annotation == annotation + + def test_relationships(self, session, cls_, visualization, user): + obj = cls_() + obj.user = user + obj.visualization = visualization + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.visualization.id == visualization.id + assert stored_obj.user.id == user.id -def test_VisualizationRatingAssociation(model, session, visualization, user): - cls = model.VisualizationRatingAssociation - assert cls.__tablename__ == 'visualization_rating_association' - with dbcleanup(session, cls): +class TestVisualizationRatingAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'visualization_rating_association' + + def test_columns(self, session, cls_, visualization, user): rating = 9 - obj = cls(user, visualization, rating) - obj_id = persist(session, obj) + obj = cls_(user, visualization, rating) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.visualization_id == visualization.id - assert stored_obj.user_id == user.id - assert stored_obj.rating == rating - # test mapped relationships - assert stored_obj.visualization.id == visualization.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.visualization_id == visualization.id + assert stored_obj.user_id == user.id + assert stored_obj.rating == rating + + def test_relationships(self, session, cls_, visualization, user): + obj = cls_(user, visualization, 1) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.visualization.id == visualization.id + assert stored_obj.user.id == user.id -def test_VisualizationRevision(model, session, visualization): - cls = model.VisualizationRevision - assert cls.__tablename__ == 'visualization_revision' - assert has_index(cls.__table__, ('dbkey',)) - with dbcleanup(session, cls): +class TestVisualizationRevision(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'visualization_revision' + assert has_index(cls_.__table__, ('dbkey',)) + + def test_columns(self, session, cls_, visualization): visualization, title, dbkey, config = visualization, 'a', 'b', 'c' - obj = cls(visualization, title, dbkey, config) create_time = datetime.now() update_time = create_time + timedelta(hours=1) + obj = cls_(visualization, title, dbkey, config) obj.create_time = create_time obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.visualization_id == visualization.id - assert stored_obj.title == title - assert stored_obj.dbkey == dbkey - assert stored_obj.config == config - # test mapped relationships - assert stored_obj.visualization.id == visualization.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.visualization_id == visualization.id + assert stored_obj.title == title + assert stored_obj.dbkey == dbkey + assert stored_obj.config == config + + def test_relationships(self, session, cls_, visualization): + obj = cls_(visualization) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.visualization.id == visualization.id -def test_WorkerProcess(model, session): - cls = model.WorkerProcess - assert cls.__tablename__ == 'worker_process' - assert has_unique_constraint(cls.__table__, ('server_name', 'hostname')) - with dbcleanup(session, cls): +class TestWorkerProcess(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'worker_process' + assert has_unique_constraint(cls_.__table__, ('server_name', 'hostname')) + + def test_columns(self, session, cls_): server_name, hostname = get_random_string(), 'a' - obj = cls(server_name, hostname) update_time = datetime.now() + obj = cls_(server_name, hostname) obj.update_time = update_time - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.server_name == server_name - assert stored_obj.hostname == hostname - assert stored_obj.pid is None - assert stored_obj.update_time == update_time + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.server_name == server_name + assert stored_obj.hostname == hostname + assert stored_obj.pid is None + assert stored_obj.update_time == update_time -def test_WorkflowStepAnnotationAssociation(model, session, workflow_step, user): - cls = model.WorkflowStepAnnotationAssociation - assert cls.__tablename__ == 'workflow_step_annotation_association' - assert has_index(cls.__table__, ('annotation',)) - with dbcleanup(session, cls): +class TestWorkflowStepAnnotationAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_step_annotation_association' + assert has_index(cls_.__table__, ('annotation',)) + + def test_columns(self, session, cls_, workflow_step, user): annotation = 'a' - obj = cls() + obj = cls_() obj.user = user obj.workflow_step = workflow_step obj.annotation = annotation - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.workflow_step_id == workflow_step.id - assert stored_obj.user_id == user.id - assert stored_obj.annotation == annotation - # test mapped relationships - assert stored_obj.workflow_step.id == workflow_step.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.user_id == user.id + assert stored_obj.annotation == annotation + + def test_relationships(self, session, cls_, workflow_step, user): + obj = cls_() + obj.user = user + obj.workflow_step = workflow_step + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.user.id == user.id -def test_WorkflowStepTagAssociation(model, session, workflow_step, tag, user): - cls = model.WorkflowStepTagAssociation - assert cls.__tablename__ == 'workflow_step_tag_association' - with dbcleanup(session, cls): +class TestWorkflowStepTagAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_step_tag_association' + + def test_columns(self, session, cls_, workflow_step, tag, user): user_tname, value, user_value = 'a', 'b', 'c' - obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) obj.user_value = user_value obj.workflow_step = workflow_step - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.workflow_step_id == workflow_step.id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - # test mapped relationships - assert stored_obj.workflow_step.id == workflow_step.id - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, workflow_step, tag, user): + obj = cls_(user=user, tag=tag) + obj.workflow_step = workflow_step + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id -def test_VisualizationTagAssociation(model, session, visualization, tag, user): - cls = model.VisualizationTagAssociation - assert cls.__tablename__ == 'visualization_tag_association' - with dbcleanup(session, cls): +class TestVisualizationTagAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'visualization_tag_association' + + def test_columns(self, session, cls_, visualization, tag, user): user_tname, value, user_value = 'a', 'b', 'c' - obj = cls(user=user, tag_id=tag.id, user_tname=user_tname, value=value) + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) obj.user_value = user_value obj.visualization = visualization - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.visualization_id == visualization.id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - # test mapped relationships - assert stored_obj.visualization.id == visualization.id - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.visualization_id == visualization.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, visualization, tag, user): + obj = cls_(user=user, tag=tag) + obj.visualization = visualization + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.visualization.id == visualization.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id -def test_WorkflowRequestInputParameter(model, session, workflow_invocation): - cls = model.WorkflowRequestInputParameter - assert cls.__tablename__ == 'workflow_request_input_parameters' - with dbcleanup(session, cls): +class TestWorkflowRequestInputParameter(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_request_input_parameters' + + def test_columns(self, session, cls_, workflow_invocation): name, value, type = 'a', 'b', 'c' - obj = cls(name, value, type) + obj = cls_(name, value, type) obj.workflow_invocation = workflow_invocation - obj_id = persist(session, obj) - stored_obj = get_stored_obj(session, cls, obj_id) - # test mapped columns - assert stored_obj.id == obj_id - assert stored_obj.workflow_invocation_id == workflow_invocation.id - assert stored_obj.name == name - assert stored_obj.value == value - assert stored_obj.type == type - # test mapped relationships - assert stored_obj.workflow_invocation.id == workflow_invocation.id + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.name == name + assert stored_obj.value == value + assert stored_obj.type == type + + def test_relationships(self, session, cls_, workflow_invocation): + obj = cls_() + obj.workflow_invocation = workflow_invocation + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id @pytest.fixture(scope='module') @@ -2576,32 +3057,56 @@ def workflow_step_tag_association(model, session): yield from dbcleanup_wrapper(session, wsta) -@contextmanager -def dbcleanup(session, cls): - """ - Ensure all records of cls are deleted from the database on exit. - """ - try: - yield - finally: - session.execute(delete(cls)) - - def dbcleanup_wrapper(session, obj): - persist(session, obj) - with dbcleanup(session, type(obj)): + with dbcleanup(session, obj): yield obj +@contextmanager +def dbcleanup(session, obj, where_clause=None): + """ + Use the session to store obj in database; delete from database on exit, bypassing the session. + + If obj does not have an id field, a SQLAlchemy WHERE clause should be provided to construct + a custom select statement. + """ + return_id = where_clause is None + + try: + obj_id = persist(session, obj, return_id) + yield obj_id + finally: + table = obj.__table__ + if where_clause is None: + where_clause = get_default_where_clause(type(obj), obj_id) + stmt = delete(table).where(where_clause) + session.execute(stmt) + + def persist(session, obj, return_id=True): + """ + Use the session to store obj in database, then remove obj from session, + so that on a subsequent load from the database we get a clean instance. + """ session.add(obj) session.flush() obj_id = obj.id if return_id else None # save this before obj is expunged - # Remove from session, so that on a subsequent load we get a *new* obj from the db session.expunge(obj) return obj_id +def delete_from_database(session, objects): + """ + Delete each object in objects from database. + May be called at the end of a test if use of a context manager is impractical. + (Assume all objects have the id field as their primary key.) + """ + for obj in objects: + table = obj.__table__ + stmt = delete(table).where(table.c.id == obj.id) + session.execute(stmt) + + def has_unique_constraint(table, fields): for constraint in table.constraints: if isinstance(constraint, UniqueConstraint): @@ -2617,6 +3122,21 @@ def has_index(table, fields): return True +def get_stored_obj2(session, cls, obj_id=None, where_clause=None): + # Either obj_id or where_clause must be provided, but not both + assert bool(obj_id) ^ (where_clause is not None) + + if where_clause is None: + where_clause = get_default_where_clause(cls, obj_id) + stmt = select(cls).where(where_clause) + return session.execute(stmt).scalar_one() + + +def get_default_where_clause(cls, obj_id): + where_clause = cls.__table__.c.id == obj_id + return where_clause + + def get_stored_obj(session, cls, obj_id=None, stmt=None): # Either obj_id or stmt must be provided, but not both assert bool(obj_id) ^ (stmt is not None) From 73b88c5c2bee427bce93273110bd7870f469b856 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 21 Jul 2021 16:21:08 -0400 Subject: [PATCH 110/225] Generalize stored object retrieval Just pass a WHERE clause defaulting to None. --- test/unit/model/test_model.py | 41 ++++++++++++++--------------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 8a5e785e9ef..b19b6c5f477 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -570,7 +570,7 @@ class TestHistoryAudit(BaseTest): where_clause = and_(cls_.history_id == history.id, cls_.update_time == update_time) with dbcleanup(session, obj, where_clause): - stored_obj = get_stored_obj2(session, cls_, where_clause=where_clause) + stored_obj = get_stored_obj(session, cls_, where_clause=where_clause) # test columns assert stored_obj.history_id == history.id assert stored_obj.update_time == update_time @@ -1817,7 +1817,7 @@ class TestPasswordResetToken(BaseTest): where_clause = cls_.token == token with dbcleanup(session, obj, where_clause): - stored_obj = get_stored_obj2(session, cls_, where_clause=where_clause) + stored_obj = get_stored_obj(session, cls_, where_clause=where_clause) # test columns assert stored_obj.token == token assert stored_obj.expiration_time == expiration_time @@ -3107,6 +3107,20 @@ def delete_from_database(session, objects): session.execute(stmt) +def get_stored_obj(session, cls, obj_id=None, where_clause=None): + # Either obj_id or where_clause must be provided, but not both + assert bool(obj_id) ^ (where_clause is not None) + if where_clause is None: + where_clause = get_default_where_clause(cls, obj_id) + stmt = select(cls).where(where_clause) + return session.execute(stmt).scalar_one() + + +def get_default_where_clause(cls, obj_id): + where_clause = cls.__table__.c.id == obj_id + return where_clause + + def has_unique_constraint(table, fields): for constraint in table.constraints: if isinstance(constraint, UniqueConstraint): @@ -3122,29 +3136,6 @@ def has_index(table, fields): return True -def get_stored_obj2(session, cls, obj_id=None, where_clause=None): - # Either obj_id or where_clause must be provided, but not both - assert bool(obj_id) ^ (where_clause is not None) - - if where_clause is None: - where_clause = get_default_where_clause(cls, obj_id) - stmt = select(cls).where(where_clause) - return session.execute(stmt).scalar_one() - - -def get_default_where_clause(cls, obj_id): - where_clause = cls.__table__.c.id == obj_id - return where_clause - - -def get_stored_obj(session, cls, obj_id=None, stmt=None): - # Either obj_id or stmt must be provided, but not both - assert bool(obj_id) ^ (stmt is not None) - if stmt is None: - stmt = select(cls).where(cls.id == obj_id) - return session.execute(stmt).scalar_one() - - def get_random_string(): """Generate unique values to accommodate unique constraints.""" return str(random.random()) From 89481363e3b32ed08e931030126e559f90324c2e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 22 Jul 2021 13:05:41 -0400 Subject: [PATCH 111/225] Remove redundant column names --- lib/galaxy/model/__init__.py | 1010 +++++++++++++++++----------------- 1 file changed, 500 insertions(+), 510 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 8b06c95546f..b25742bcdea 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -296,11 +296,11 @@ class WorkerProcess(Base, UsesCreateAndUpdateTime, _HasTable): UniqueConstraint('server_name', 'hostname'), ) - id = Column("id", Integer, primary_key=True) - server_name = Column("server_name", String(255), index=True) - hostname = Column("hostname", String(255)) - pid = Column("pid", Integer) - update_time = Column("update_time", DateTime, default=now, onupdate=now) + id = Column(Integer, primary_key=True) + server_name = Column(String(255), index=True) + hostname = Column(String(255)) + pid = Column(Integer) + update_time = Column(DateTime, default=now, onupdate=now) def __init__(self, server_name, hostname): self.server_name = server_name @@ -689,9 +689,9 @@ class User(Dictifiable, RepresentById): class PasswordResetToken(Base, _HasTable): __tablename__ = 'password_reset_token' - token = Column('token', String(32), primary_key=True, unique=True, index=True) - expiration_time = Column('expiration_time', DateTime) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + token = Column(String(32), primary_key=True, unique=True, index=True) + expiration_time = Column(DateTime) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) user = relationship('User', back_populates='reset_tokens') def __init__(self, user, token=None): @@ -706,18 +706,18 @@ class PasswordResetToken(Base, _HasTable): class DynamicTool(Base, Dictifiable, RepresentById): __tablename__ = 'dynamic_tool' - id = Column('id', Integer, primary_key=True) - uuid = Column('uuid', UUIDType()) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, index=True, default=now, onupdate=now) - tool_id = Column('tool_id', Unicode(255)) - tool_version = Column('tool_version', Unicode(255)) - tool_format = Column('tool_format', Unicode(255)) - tool_path = Column('tool_path', Unicode(255)) - tool_directory = Column('tool_directory', Unicode(255)) - hidden = Column('hidden', Boolean, default=True) - active = Column('active', Boolean, default=True) - value = Column('value', MutableJSONType) + id = Column(Integer, primary_key=True) + uuid = Column(UUIDType()) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, index=True, default=now, onupdate=now) + tool_id = Column(Unicode(255)) + tool_version = Column(Unicode(255)) + tool_format = Column(Unicode(255)) + tool_path = Column(Unicode(255)) + tool_directory = Column(Unicode(255)) + hidden = Column(Boolean, default=True) + active = Column(Boolean, default=True) + value = Column(MutableJSONType) workflow_steps = relationship('WorkflowStep', back_populates='dynamic_tool') dict_collection_visible_keys = ('id', 'tool_id', 'tool_format', 'tool_version', 'uuid', 'active', 'hidden') @@ -750,44 +750,44 @@ class BaseJobMetric: class JobMetricText(Base, BaseJobMetric, RepresentById): __tablename__ = 'job_metric_text' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - plugin = Column('plugin', Unicode(255)) - metric_name = Column('metric_name', Unicode(255)) - metric_value = Column('metric_value', Unicode(JOB_METRIC_MAX_LENGTH)) + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + plugin = Column(Unicode(255)) + metric_name = Column(Unicode(255)) + metric_value = Column(Unicode(JOB_METRIC_MAX_LENGTH)) job = relationship('Job', back_populates='text_metrics') class JobMetricNumeric(Base, BaseJobMetric, RepresentById): __tablename__ = 'job_metric_numeric' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - plugin = Column('plugin', Unicode(255)) - metric_name = Column('metric_name', Unicode(255)) - metric_value = Column('metric_value', Numeric(JOB_METRIC_PRECISION, JOB_METRIC_SCALE)) + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + plugin = Column(Unicode(255)) + metric_name = Column(Unicode(255)) + metric_value = Column(Numeric(JOB_METRIC_PRECISION, JOB_METRIC_SCALE)) job = relationship('Job', back_populates='numeric_metrics') class TaskMetricText(Base, BaseJobMetric, RepresentById): __tablename__ = 'task_metric_text' - id = Column('id', Integer, primary_key=True) - task_id = Column('task_id', Integer, ForeignKey('task.id'), index=True) - plugin = Column('plugin', Unicode(255)) - metric_name = Column('metric_name', Unicode(255)) - metric_value = Column('metric_value', Unicode(JOB_METRIC_MAX_LENGTH)) + id = Column(Integer, primary_key=True) + task_id = Column(Integer, ForeignKey('task.id'), index=True) + plugin = Column(Unicode(255)) + metric_name = Column(Unicode(255)) + metric_value = Column(Unicode(JOB_METRIC_MAX_LENGTH)) task = relationship('Task', back_populates='text_metrics') class TaskMetricNumeric(Base, BaseJobMetric, RepresentById): __tablename__ = 'task_metric_numeric' - id = Column('id', Integer, primary_key=True) - task_id = Column('task_id', Integer, ForeignKey('task.id'), index=True) - plugin = Column('plugin', Unicode(255)) - metric_name = Column('metric_name', Unicode(255)) - metric_value = Column('metric_value', Numeric(JOB_METRIC_PRECISION, JOB_METRIC_SCALE)) + id = Column(Integer, primary_key=True) + task_id = Column(Integer, ForeignKey('task.id'), index=True) + plugin = Column(Unicode(255)) + metric_name = Column(Unicode(255)) + metric_value = Column(Numeric(JOB_METRIC_PRECISION, JOB_METRIC_SCALE)) task = relationship('Task', back_populates='numeric_metrics') @@ -1438,27 +1438,27 @@ class Task(Base, JobLike, RepresentById): """ __tablename__ = 'task' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - execution_time = Column('execution_time', DateTime) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - state = Column('state', String(64), index=True) - command_line = Column('command_line', TEXT) - param_filename = Column('param_filename', String(1024)) - runner_name = Column('runner_name', String(255)) - job_stdout = Column('job_stdout', TEXT) # job_stdout makes sense here because it is short for job script standard out - job_stderr = Column('job_stderr', TEXT) - tool_stdout = Column('tool_stdout', TEXT) - tool_stderr = Column('tool_stderr', TEXT) - exit_code = Column('exit_code', Integer, nullable=True) - job_messages = Column('job_messages', MutableJSONType, nullable=True) - info = Column('info', TrimmedString(255)) - traceback = Column('traceback', TEXT) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True, nullable=False) - working_directory = Column('working_directory', String(1024)) - task_runner_name = Column('task_runner_name', String(255)) - task_runner_external_id = Column('task_runner_external_id', String(255)) - prepare_input_files_cmd = Column('prepare_input_files_cmd', TEXT) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + execution_time = Column(DateTime) + update_time = Column(DateTime, default=now, onupdate=now) + state = Column(String(64), index=True) + command_line = Column(TEXT) + param_filename = Column(String(1024)) + runner_name = Column(String(255)) + job_stdout = Column(TEXT) # job_stdout makes sense here because it is short for job script standard out + job_stderr = Column(TEXT) + tool_stdout = Column(TEXT) + tool_stderr = Column(TEXT) + exit_code = Column(Integer, nullable=True) + job_messages = Column(MutableJSONType, nullable=True) + info = Column(TrimmedString(255)) + traceback = Column(TEXT) + job_id = Column(Integer, ForeignKey('job.id'), index=True, nullable=False) + working_directory = Column(String(1024)) + task_runner_name = Column(String(255)) + task_runner_external_id = Column(String(255)) + prepare_input_files_cmd = Column(TEXT) job = relationship('Job', back_populates='tasks') text_metrics = relationship('TaskMetricText', back_populates='task') numeric_metrics = relationship('TaskMetricNumeric', back_populates='task') @@ -1613,10 +1613,10 @@ class Task(Base, JobLike, RepresentById): class JobParameter(Base, RepresentById): __tablename__ = 'job_parameter' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - name = Column('name', String(255)) - value = Column('value', TEXT) + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + name = Column(String(255)) + value = Column(TEXT) def __init__(self, name, value): self.name = name @@ -1629,12 +1629,12 @@ class JobParameter(Base, RepresentById): class JobToInputDatasetAssociation(Base, RepresentById): __tablename__ = 'job_to_input_dataset' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - dataset_id = Column('dataset_id', Integer, + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + dataset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) - dataset_version = Column('dataset_version', Integer) - name = Column('name', String(255)) + dataset_version = Column(Integer) + name = Column(String(255)) dataset = relationship('HistoryDatasetAssociation', lazy=False, back_populates='dependent_jobs') job = relationship('Job', back_populates='input_datasets') @@ -1647,11 +1647,10 @@ class JobToInputDatasetAssociation(Base, RepresentById): class JobToOutputDatasetAssociation(Base, RepresentById): __tablename__ = 'job_to_output_dataset' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - dataset_id = Column('dataset_id', Integer, - ForeignKey('history_dataset_association.id'), index=True) - name = Column('name', String(255)) + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + dataset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + name = Column(String(255)) dataset = relationship('HistoryDatasetAssociation', lazy=False, back_populates='creating_job_associations') job = relationship('Job', back_populates='output_datasets') @@ -1663,12 +1662,11 @@ class JobToOutputDatasetAssociation(Base, RepresentById): class JobToInputDatasetCollectionAssociation(Base, RepresentById): __tablename__ = 'job_to_input_dataset_collection' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - dataset_collection_id = Column('dataset_collection_id', Integer, - ForeignKey('history_dataset_collection_association.id'), index=True) - dataset_version = Column('dataset_version', Integer) - name = Column('name', String(255)) + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + dataset_version = Column(Integer) + name = Column(String(255)) dataset_collection = relationship('HistoryDatasetCollectionAssociation', lazy=False) job = relationship('Job', back_populates='input_dataset_collections') @@ -1680,10 +1678,10 @@ class JobToInputDatasetCollectionAssociation(Base, RepresentById): class JobToInputDatasetCollectionElementAssociation(Base, RepresentById): __tablename__ = 'job_to_input_dataset_collection_element' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - dataset_collection_element_id = Column('dataset_collection_element_id', Integer, ForeignKey('dataset_collection_element.id'), index=True) - name = Column('name', Unicode(255)) + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + dataset_collection_element_id = Column(Integer, ForeignKey('dataset_collection_element.id'), index=True) + name = Column(Unicode(255)) dataset_collection_element = relationship('DatasetCollectionElement', lazy=False) job = relationship('Job', back_populates='input_dataset_collection_elements') @@ -1697,10 +1695,10 @@ class JobToInputDatasetCollectionElementAssociation(Base, RepresentById): class JobToOutputDatasetCollectionAssociation(Base, RepresentById): __tablename__ = 'job_to_output_dataset_collection' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - dataset_collection_id = Column('dataset_collection_id', Integer, ForeignKey('history_dataset_collection_association.id'), index=True) - name = Column('name', Unicode(255)) + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + name = Column(Unicode(255)) dataset_collection_instance = relationship( 'HistoryDatasetCollectionAssociation', lazy=False, @@ -1718,11 +1716,10 @@ class JobToOutputDatasetCollectionAssociation(Base, RepresentById): class JobToImplicitOutputDatasetCollectionAssociation(Base, RepresentById): __tablename__ = 'job_to_implicit_output_dataset_collection' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - dataset_collection_id = Column( - 'dataset_collection_id', Integer, ForeignKey('dataset_collection.id'), index=True) - name = Column('name', Unicode(255)) + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + dataset_collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True) + name = Column(Unicode(255)) dataset_collection = relationship( 'DatasetCollection', back_populates="output_dataset_collections") job = relationship('Job', back_populates='output_dataset_collections') @@ -1735,10 +1732,10 @@ class JobToImplicitOutputDatasetCollectionAssociation(Base, RepresentById): class JobToInputLibraryDatasetAssociation(Base, RepresentById): __tablename__ = 'job_to_input_library_dataset' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - ldda_id = Column('ldda_id', Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) - name = Column('name', Unicode(255)) + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + ldda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + name = Column(Unicode(255)) job = relationship('Job', back_populates='input_library_datasets') dataset = relationship( 'LibraryDatasetDatasetAssociation', lazy=False, back_populates='dependent_jobs') @@ -1751,10 +1748,10 @@ class JobToInputLibraryDatasetAssociation(Base, RepresentById): class JobToOutputLibraryDatasetAssociation(Base, RepresentById): __tablename__ = 'job_to_output_library_dataset' - id = Column('id', Integer, primary_key=True) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - ldda_id = Column('ldda_id', Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) - name = Column('name', Unicode(255)) + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + ldda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + name = Column(Unicode(255)) job = relationship('Job', back_populates='output_library_datasets') dataset = relationship( 'LibraryDatasetDatasetAssociation', lazy=False, back_populates='creating_job_associations') @@ -1767,12 +1764,12 @@ class JobToOutputLibraryDatasetAssociation(Base, RepresentById): class JobStateHistory(Base, RepresentById): __tablename__ = 'job_state_history' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - job_id = Column('job_id', Integer, ForeignKey('job.id'), index=True) - state = Column('state', String(64), index=True) - info = Column('info', TrimmedString(255)) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + state = Column(String(64), index=True) + info = Column(TrimmedString(255)) job = relationship('Job', back_populates='state_history') def __init__(self, job): @@ -2062,11 +2059,11 @@ class DeferredJob(RepresentById): class Group(Base, Dictifiable, RepresentById): __tablename__ = 'galaxy_group' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - name = Column('name', String(255), index=True, unique=True) - deleted = Column('deleted', Boolean, index=True, default=False) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + name = Column(String(255), index=True, unique=True) + deleted = Column(Boolean, index=True, default=False) quotas = relationship('GroupQuotaAssociation', back_populates='group') roles = relationship('GroupRoleAssociation', back_populates='group') users = relationship('UserGroupAssociation', back_populates='group') @@ -2082,11 +2079,11 @@ class Group(Base, Dictifiable, RepresentById): class UserGroupAssociation(Base, RepresentById): __tablename__ = 'user_group_association' - id = Column('id', Integer, primary_key=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - group_id = Column('group_id', Integer, ForeignKey('galaxy_group.id'), index=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + group_id = Column(Integer, ForeignKey('galaxy_group.id'), index=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) user = relationship('User', back_populates='groups') group = relationship('Group', back_populates='users') @@ -2105,8 +2102,8 @@ class HistoryAudit(Base, RepresentById): PrimaryKeyConstraint(sqlite_on_conflict='IGNORE'), ) - history_id = Column('history_id', Integer, ForeignKey('history.id'), primary_key=True, nullable=False) - update_time = Column('update_time', DateTime, default=now, primary_key=True, nullable=False) + history_id = Column(Integer, ForeignKey('history.id'), primary_key=True, nullable=False) + update_time = Column(DateTime, default=now, primary_key=True, nullable=False) history = relationship('History') def __init__(self, history, update_time): @@ -2533,9 +2530,9 @@ class UserShareAssociation(RepresentById): class HistoryUserShareAssociation(Base, UserShareAssociation): __tablename__ = 'history_user_share_association' - id = Column('id', Integer, primary_key=True) - history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) + id = Column(Integer, primary_key=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) user = relationship('User', back_populates='histories_shared_by_others') history = relationship('History', back_populates='users_shared_with') @@ -2553,11 +2550,11 @@ class UserRoleAssociation(RepresentById): class GroupRoleAssociation(Base, RepresentById): __tablename__ = 'group_role_association' - id = Column('id', Integer, primary_key=True) - group_id = Column('group_id', Integer, ForeignKey('galaxy_group.id'), index=True) - role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) + id = Column(Integer, primary_key=True) + group_id = Column(Integer, ForeignKey('galaxy_group.id'), index=True) + role_id = Column(Integer, ForeignKey('role.id'), index=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) group = relationship('Group', back_populates='roles') role = relationship('Role', back_populates='groups') @@ -2569,13 +2566,13 @@ class GroupRoleAssociation(Base, RepresentById): class Role(Base, Dictifiable, RepresentById): __tablename__ = 'role' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - name = Column('name', String(255), index=True, unique=True) - description = Column('description', TEXT) - type = Column('type', String(40), index=True) - deleted = Column('deleted', Boolean, index=True, default=False) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + name = Column(String(255), index=True, unique=True) + description = Column(TEXT) + type = Column(String(40), index=True) + deleted = Column(Boolean, index=True, default=False) dataset_actions = relationship('DatasetPermissions', back_populates='role') groups = relationship('GroupRoleAssociation', back_populates='role') library_actions = relationship('LibraryPermissions', back_populates='role') @@ -2605,11 +2602,11 @@ class Role(Base, Dictifiable, RepresentById): class UserQuotaAssociation(Base, Dictifiable, RepresentById): __tablename__ = 'user_quota_association' - id = Column('id', Integer, primary_key=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - quota_id = Column('quota_id', Integer, ForeignKey('quota.id'), index=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + quota_id = Column(Integer, ForeignKey('quota.id'), index=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) user = relationship('User', back_populates='quotas') quota = relationship('Quota', back_populates='users') @@ -2623,11 +2620,11 @@ class UserQuotaAssociation(Base, Dictifiable, RepresentById): class GroupQuotaAssociation(Base, Dictifiable, RepresentById): __tablename__ = 'group_quota_association' - id = Column('id', Integer, primary_key=True) - group_id = Column("group_id", Integer, ForeignKey("galaxy_group.id"), index=True) - quota_id = Column('quota_id', Integer, ForeignKey('quota.id'), index=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) + id = Column(Integer, primary_key=True) + group_id = Column(Integer, ForeignKey("galaxy_group.id"), index=True) + quota_id = Column(Integer, ForeignKey('quota.id'), index=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) group = relationship('Group', back_populates='quotas') quota = relationship('Quota', back_populates='groups') @@ -2641,14 +2638,14 @@ class GroupQuotaAssociation(Base, Dictifiable, RepresentById): class Quota(Base, Dictifiable, RepresentById): __tablename__ = 'quota' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - name = Column('name', String(255), index=True, unique=True) - description = Column('description', TEXT) - bytes = Column('bytes', BigInteger) - operation = Column('operation', String(8)) - deleted = Column('deleted', Boolean, index=True, default=False) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + name = Column(String(255), index=True, unique=True) + description = Column(TEXT) + bytes = Column(BigInteger) + operation = Column(String(8)) + deleted = Column(Boolean, index=True, default=False) default = relationship('DefaultQuotaAssociation', back_populates='quota') groups = relationship('GroupQuotaAssociation', back_populates='quota') users = relationship('UserQuotaAssociation', back_populates='quota') @@ -2689,11 +2686,11 @@ class Quota(Base, Dictifiable, RepresentById): class DefaultQuotaAssociation(Base, Dictifiable, RepresentById): __tablename__ = 'default_quota_association' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - type = Column('type', String(32), index=True, unique=True) - quota_id = Column('quota_id', Integer, ForeignKey('quota.id'), index=True) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + type = Column(String(32), index=True, unique=True) + quota_id = Column(Integer, ForeignKey('quota.id'), index=True) quota = relationship('Quota', back_populates='default') dict_element_visible_keys = ['type'] @@ -2711,12 +2708,12 @@ class DefaultQuotaAssociation(Base, Dictifiable, RepresentById): class DatasetPermissions(Base, RepresentById): __tablename__ = 'dataset_permissions' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - action = Column('action', TEXT) - dataset_id = Column('dataset_id', Integer, ForeignKey('dataset.id'), index=True) - role_id = Column('role_id', Integer, ForeignKey("role.id"), index=True) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + action = Column(TEXT) + dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + role_id = Column(Integer, ForeignKey("role.id"), index=True) dataset = relationship('Dataset', back_populates='actions') role = relationship('Role', back_populates='dataset_actions') @@ -2732,12 +2729,12 @@ class DatasetPermissions(Base, RepresentById): class LibraryPermissions(Base, RepresentById): __tablename__ = 'library_permissions' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - action = Column('action', TEXT) - library_id = Column('library_id', Integer, ForeignKey('library.id'), nullable=True, index=True) - role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + action = Column(TEXT) + library_id = Column(Integer, ForeignKey('library.id'), nullable=True, index=True) + role_id = Column(Integer, ForeignKey('role.id'), index=True) library = relationship('Library', back_populates='actions') role = relationship('Role', back_populates='library_actions') @@ -2753,12 +2750,12 @@ class LibraryPermissions(Base, RepresentById): class LibraryFolderPermissions(Base, RepresentById): __tablename__ = 'library_folder_permissions' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - action = Column('action', TEXT) - library_folder_id = Column('library_folder_id', Integer, ForeignKey('library_folder.id'), nullable=True, index=True) - role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + action = Column(TEXT) + library_folder_id = Column(Integer, ForeignKey('library_folder.id'), nullable=True, index=True) + role_id = Column(Integer, ForeignKey('role.id'), index=True) folder = relationship('LibraryFolder', back_populates='actions') role = relationship('Role', back_populates='library_folder_actions') @@ -2774,13 +2771,12 @@ class LibraryFolderPermissions(Base, RepresentById): class LibraryDatasetPermissions(Base, RepresentById): __tablename__ = 'library_dataset_permissions' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - action = Column('action', TEXT) - library_dataset_id = Column('library_dataset_id', Integer, - ForeignKey('library_dataset.id'), nullable=True, index=True) - role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + action = Column(TEXT) + library_dataset_id = Column(Integer, ForeignKey('library_dataset.id'), nullable=True, index=True) + role_id = Column(Integer, ForeignKey('role.id'), index=True) library_dataset = relationship('LibraryDataset', back_populates='actions') role = relationship('Role', back_populates='library_dataset_actions') @@ -2796,13 +2792,13 @@ class LibraryDatasetPermissions(Base, RepresentById): class LibraryDatasetDatasetAssociationPermissions(Base, RepresentById): __tablename__ = 'library_dataset_dataset_association_permissions' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - action = Column('action', TEXT) - library_dataset_dataset_association_id = Column('library_dataset_dataset_association_id', + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + action = Column(TEXT) + library_dataset_dataset_association_id = Column( Integer, ForeignKey('library_dataset_dataset_association.id'), nullable=True, index=True) - role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + role_id = Column(Integer, ForeignKey('role.id'), index=True) library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', back_populates='actions') role = relationship('Role', back_populates='library_dataset_dataset_actions') @@ -2818,10 +2814,10 @@ class LibraryDatasetDatasetAssociationPermissions(Base, RepresentById): class DefaultUserPermissions(Base, RepresentById): __tablename__ = 'default_user_permissions' - id = Column('id', Integer, primary_key=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - action = Column('action', TEXT) - role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + action = Column(TEXT) + role_id = Column(Integer, ForeignKey('role.id'), index=True) user = relationship('User', back_populates='default_permissions') role = relationship('Role') @@ -2834,10 +2830,10 @@ class DefaultUserPermissions(Base, RepresentById): class DefaultHistoryPermissions(Base, RepresentById): __tablename__ = 'default_history_permissions' - id = Column('id', Integer, primary_key=True) - history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) - action = Column('action', TEXT) - role_id = Column('role_id', Integer, ForeignKey('role.id'), index=True) + id = Column(Integer, primary_key=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + action = Column(TEXT) + role_id = Column(Integer, ForeignKey('role.id'), index=True) history = relationship('History', back_populates='default_permissions') role = relationship('Role') @@ -3135,11 +3131,11 @@ class Dataset(StorableObject, RepresentById, _HasTable): class DatasetSource(Base, RepresentById): __tablename__ = 'dataset_source' - id = Column('id', Integer, primary_key=True) - dataset_id = Column('dataset_id', Integer, ForeignKey('dataset.id'), index=True) - source_uri = Column("source_uri", TEXT) - extra_files_path = Column("extra_files_path", TEXT) - transform = Column("transform", MutableJSONType) + id = Column(Integer, primary_key=True) + dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + source_uri = Column(TEXT) + extra_files_path = Column(TEXT) + transform = Column(MutableJSONType) dataset = relationship('Dataset', back_populates='sources') hashes = relationship('DatasetSourceHash', back_populates='source') @@ -3147,21 +3143,21 @@ class DatasetSource(Base, RepresentById): class DatasetSourceHash(Base, RepresentById): __tablename__ = 'dataset_source_hash' - id = Column('id', Integer, primary_key=True) - dataset_source_id = Column('dataset_source_id', Integer, ForeignKey('dataset_source.id'), index=True) - hash_function = Column('hash_function', TEXT) - hash_value = Column('hash_value', TEXT) + id = Column(Integer, primary_key=True) + dataset_source_id = Column(Integer, ForeignKey('dataset_source.id'), index=True) + hash_function = Column(TEXT) + hash_value = Column(TEXT) source = relationship('DatasetSource', back_populates='hashes') class DatasetHash(Base, RepresentById): __tablename__ = 'dataset_hash' - id = Column('id', Integer, primary_key=True) - dataset_id = Column('dataset_id', Integer, ForeignKey('dataset.id'), index=True) - hash_function = Column('hash_function', TEXT) - hash_value = Column('hash_value', TEXT) - extra_files_path = Column('extra_files_path', TEXT) + id = Column(Integer, primary_key=True) + dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + hash_function = Column(TEXT) + hash_value = Column(TEXT) + extra_files_path = Column(TEXT) dataset = relationship('Dataset', back_populates='hashes') def serialize(self, id_encoder, serialization_options): @@ -4095,15 +4091,15 @@ class HistoryDatasetAssociationSubset(RepresentById): class Library(Base, Dictifiable, HasName, RepresentById): __tablename__ = 'library' - id = Column("id", Integer, primary_key=True) - root_folder_id = Column("root_folder_id", Integer, ForeignKey("library_folder.id"), index=True) - create_time = Column("create_time", DateTime, default=now) - update_time = Column("update_time", DateTime, default=now, onupdate=now) - name = Column("name", String(255), index=True) - deleted = Column("deleted", Boolean, index=True, default=False) - purged = Column("purged", Boolean, index=True, default=False) - description = Column("description", TEXT) - synopsis = Column("synopsis", TEXT) + id = Column(Integer, primary_key=True) + root_folder_id = Column(Integer, ForeignKey("library_folder.id"), index=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + name = Column(String(255), index=True) + deleted = Column(Boolean, index=True, default=False) + purged = Column(Boolean, index=True, default=False) + description = Column(TEXT) + synopsis = Column(TEXT) root_folder = relationship('LibraryFolder', back_populates='library_root') actions = relationship('LibraryPermissions', back_populates='library') @@ -4176,17 +4172,17 @@ class LibraryFolder(Base, Dictifiable, HasName, RepresentById): Index('ix_library_folder_name', 'name', mysql_length=200), ) - id = Column('id', Integer, primary_key=True) - parent_id = Column('parent_id', Integer, ForeignKey('library_folder.id'), nullable=True, index=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - name = Column('name', TEXT) - description = Column('description', TEXT) - order_id = Column('order_id', Integer) # not currently being used, but for possible future use - item_count = Column('item_count', Integer) - deleted = Column('deleted', Boolean, index=True, default=False) - purged = Column('purged', Boolean, index=True, default=False) - genome_build = Column('genome_build', TrimmedString(40)) + id = Column(Integer, primary_key=True) + parent_id = Column(Integer, ForeignKey('library_folder.id'), nullable=True, index=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + name = Column(TEXT) + description = Column(TEXT) + order_id = Column(Integer) # not currently being used, but for possible future use + item_count = Column(Integer) + deleted = Column(Boolean, index=True, default=False) + purged = Column(Boolean, index=True, default=False) + genome_build = Column(TrimmedString(40)) folders = relationship('LibraryFolder', primaryjoin=('LibraryFolder.id == LibraryFolder.parent_id'), @@ -4297,22 +4293,22 @@ class LibraryFolder(Base, Dictifiable, HasName, RepresentById): class LibraryDataset(Base, RepresentById): __tablename__ = 'library_dataset' - id = Column('id', Integer, primary_key=True) + id = Column(Integer, primary_key=True) # current version of dataset, if null, there is not a current version selected - library_dataset_dataset_association_id = Column('library_dataset_dataset_association_id', Integer, + library_dataset_dataset_association_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id', use_alter=True, name='library_dataset_dataset_association_id_fk'), nullable=True, index=True) - folder_id = Column('folder_id', Integer, ForeignKey('library_folder.id'), index=True) + folder_id = Column(Integer, ForeignKey('library_folder.id'), index=True) # not currently being used, but for possible future use - order_id = Column('order_id', Integer) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) + order_id = Column(Integer) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) # when not None/null this will supercede display in library (but not when imported into user's history?) _name = Column('name', TrimmedString(255), index=True) # when not None/null this will supercede display in library (but not when imported into user's history?) _info = Column('info', TrimmedString(255)) - deleted = Column('deleted', Boolean, index=True, default=False) - purged = Column('purged', Boolean, index=True, default=False) + deleted = Column(Boolean, index=True, default=False) + purged = Column(Boolean, index=True, default=False) folder = relationship('LibraryFolder') library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', foreign_keys=library_dataset_dataset_association_id, @@ -6403,12 +6399,12 @@ class WorkflowRequestInputParameter(Base, Dictifiable, RepresentById): """ __tablename__ = 'workflow_request_input_parameters' - id = Column('id', Integer, primary_key=True) - workflow_invocation_id = Column('workflow_invocation_id', Integer, - ForeignKey('workflow_invocation.id', onupdate='CASCADE', ondelete='CASCADE')) - name = Column('name', Unicode(255)) - value = Column('value', TEXT) - type = Column('type', Unicode(255)) + id = Column(Integer, primary_key=True) + workflow_invocation_id = Column( + Integer, ForeignKey('workflow_invocation.id', onupdate='CASCADE', ondelete='CASCADE')) + name = Column(Unicode(255)) + value = Column(TEXT) + type = Column(Unicode(255)) workflow_invocation = relationship('WorkflowInvocation', back_populates='input_parameters') dict_collection_visible_keys = ['id', 'name', 'value', 'type'] @@ -6537,16 +6533,16 @@ class MetadataFile(StorableObject, RepresentById): class FormDefinition(Base, Dictifiable, RepresentById): __tablename__ = 'form_definition' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - name = Column('name', TrimmedString(255), nullable=False) - desc = Column('desc', TEXT) - form_definition_current_id = Column('form_definition_current_id', Integer, - ForeignKey('form_definition_current.id', use_alter=True), index=True, nullable=False) - fields = Column('fields', MutableJSONType) - type = Column('type', TrimmedString(255), index=True) - layout = Column('layout', MutableJSONType) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + name = Column(TrimmedString(255), nullable=False) + desc = Column(TEXT) + form_definition_current_id = Column( + Integer, ForeignKey('form_definition_current.id', use_alter=True), index=True, nullable=False) + fields = Column(MutableJSONType) + type = Column(TrimmedString(255), index=True) + layout = Column(MutableJSONType) form_definition_current = relationship( 'FormDefinitionCurrent', back_populates='forms', @@ -6599,11 +6595,11 @@ class FormDefinition(Base, Dictifiable, RepresentById): class FormDefinitionCurrent(Base, RepresentById): __tablename__ = 'form_definition_current' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - latest_form_id = Column('latest_form_id', Integer, ForeignKey('form_definition.id'), index=True) - deleted = Column('deleted', Boolean, index=True, default=False) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + latest_form_id = Column(Integer, ForeignKey('form_definition.id'), index=True) + deleted = Column(Boolean, index=True, default=False) forms = relationship( 'FormDefinition', back_populates='form_definition_current', @@ -6621,11 +6617,11 @@ class FormDefinitionCurrent(Base, RepresentById): class FormValues(Base, RepresentById): __tablename__ = 'form_values' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - form_definition_id = Column('form_definition_id', Integer, ForeignKey('form_definition.id'), index=True) - content = Column('content', MutableJSONType) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + form_definition_id = Column(Integer, ForeignKey('form_definition.id'), index=True) + content = Column(MutableJSONType) form_definition = relationship( 'FormDefinition', primaryjoin=('FormValues.form_definition_id == FormDefinition.id')) @@ -6638,21 +6634,21 @@ class FormValues(Base, RepresentById): class UserAddress(Base, RepresentById): __tablename__ = 'user_address' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - desc = Column('desc', TrimmedString(255)) - name = Column('name', TrimmedString(255), nullable=False) - institution = Column('institution', TrimmedString(255)) - address = Column('address', TrimmedString(255), nullable=False) - city = Column('city', TrimmedString(255), nullable=False) - state = Column('state', TrimmedString(255), nullable=False) - postal_code = Column('postal_code', TrimmedString(255), nullable=False) - country = Column('country', TrimmedString(255), nullable=False) - phone = Column('phone', TrimmedString(255)) - deleted = Column('deleted', Boolean, index=True, default=False) - purged = Column('purged', Boolean, index=True, default=False) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + desc = Column(TrimmedString(255)) + name = Column(TrimmedString(255), nullable=False) + institution = Column(TrimmedString(255)) + address = Column(TrimmedString(255), nullable=False) + city = Column(TrimmedString(255), nullable=False) + state = Column(TrimmedString(255), nullable=False) + postal_code = Column(TrimmedString(255), nullable=False) + country = Column(TrimmedString(255), nullable=False) + phone = Column(TrimmedString(255)) + deleted = Column(Boolean, index=True, default=False) + purged = Column(Boolean, index=True, default=False) # `desc` needs to be fully qualified because it is shadowed by `desc` Column defined above # TODO: db migration to rename column, then use `desc` user = relationship('User', back_populates='addresses', order_by=sqlalchemy.desc('update_time')) @@ -6687,13 +6683,13 @@ class UserAddress(Base, RepresentById): class PSAAssociation(Base, AssociationMixin, RepresentById): __tablename__ = 'psa_association' - id = Column('id', Integer, primary_key=True) - server_url = Column('server_url', VARCHAR(255)) - handle = Column('handle', VARCHAR(255)) - secret = Column('secret', VARCHAR(255)) - issued = Column('issued', Integer) - lifetime = Column('lifetime', Integer) - assoc_type = Column('assoc_type', VARCHAR(64)) + id = Column(Integer, primary_key=True) + server_url = Column(VARCHAR(255)) + handle = Column(VARCHAR(255)) + secret = Column(VARCHAR(255)) + issued = Column(Integer) + lifetime = Column(Integer) + assoc_type = Column(VARCHAR(64)) # This static property is set at: galaxy.authnz.psa_authnz.PSAAuthnz sa_session = None @@ -6738,9 +6734,9 @@ class PSACode(Base, CodeMixin, RepresentById): UniqueConstraint('code', 'email'), ) - id = Column('id', Integer, primary_key=True) - email = Column('email', VARCHAR(200)) - code = Column('code', VARCHAR(32)) + id = Column(Integer, primary_key=True) + email = Column(VARCHAR(200)) + code = Column(VARCHAR(32)) # This static property is set at: galaxy.authnz.psa_authnz.PSAAuthnz sa_session = None @@ -6761,10 +6757,10 @@ class PSACode(Base, CodeMixin, RepresentById): class PSANonce(Base, NonceMixin, RepresentById): __tablename__ = 'psa_nonce' - id = Column('id', Integer, primary_key=True) - server_url = Column('server_url', VARCHAR(255)) - timestamp = Column('timestamp', Integer) - salt = Column('salt', VARCHAR(40)) + id = Column(Integer, primary_key=True) + server_url = Column(VARCHAR(255)) + timestamp = Column(Integer) + salt = Column(VARCHAR(40)) # This static property is set at: galaxy.authnz.psa_authnz.PSAAuthnz sa_session = None @@ -6792,11 +6788,11 @@ class PSANonce(Base, NonceMixin, RepresentById): class PSAPartial(Base, PartialMixin, RepresentById): __tablename__ = 'psa_partial' - id = Column('id', Integer, primary_key=True) - token = Column('token', VARCHAR(32)) - data = Column('data', TEXT) - next_step = Column('next_step', Integer) - backend = Column('backend', VARCHAR(32)) + id = Column(Integer, primary_key=True) + token = Column(VARCHAR(32)) + data = Column(TEXT) + next_step = Column(Integer) + backend = Column(VARCHAR(32)) # This static property is set at: galaxy.authnz.psa_authnz.PSAAuthnz sa_session = None @@ -6826,13 +6822,13 @@ class UserAuthnzToken(Base, UserMixin, RepresentById): __tablename__ = 'oidc_user_authnz_tokens' __table_args__ = (UniqueConstraint('provider', 'uid'),) - id = Column('id', Integer, primary_key=True) - user_id = Column('user_id', Integer, ForeignKey("galaxy_user.id"), index=True) - uid = Column('uid', VARCHAR(255)) - provider = Column('provider', VARCHAR(32)) - extra_data = Column('extra_data', MutableJSONType, nullable=True) - lifetime = Column('lifetime', Integer) - assoc_type = Column('assoc_type', VARCHAR(64)) + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + uid = Column(VARCHAR(255)) + provider = Column(VARCHAR(32)) + extra_data = Column(MutableJSONType, nullable=True) + lifetime = Column(Integer) + assoc_type = Column(VARCHAR(64)) cloudauthz = relationship('CloudAuthz', back_populates='authn') user = relationship('User', back_populates='social_auth') @@ -6946,15 +6942,15 @@ class CustosAuthnzToken(Base, RepresentById): UniqueConstraint('external_user_id', 'provider'), ) - id = Column('id', Integer, primary_key=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id')) - external_user_id = Column('external_user_id', String(64)) - provider = Column('provider', String(255)) - access_token = Column('access_token', Text) - id_token = Column('id_token', Text) - refresh_token = Column('refresh_token', Text) - expiration_time = Column('expiration_time', DateTime) - refresh_expiration_time = Column('refresh_expiration_time', DateTime) + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id')) + external_user_id = Column(String(64)) + provider = Column(String(255)) + access_token = Column(Text) + id_token = Column(Text) + refresh_token = Column(Text) + expiration_time = Column(DateTime) + refresh_expiration_time = Column(DateTime) user = relationship('User', back_populates='custos_auth') def __init__(self, user, external_user_id, provider, access_token, id_token, refresh_token, expiration_time, refresh_expiration_time): @@ -6971,16 +6967,16 @@ class CustosAuthnzToken(Base, RepresentById): class CloudAuthz(Base, _HasTable): __tablename__ = 'cloudauthz' - id = Column('id', Integer, primary_key=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - provider = Column('provider', String(255)) - config = Column('config', MutableJSONType) - authn_id = Column('authn_id', Integer, ForeignKey('oidc_user_authnz_tokens.id'), index=True) - tokens = Column('tokens', MutableJSONType) - last_update = Column('last_update', DateTime) - last_activity = Column('last_activity', DateTime) - description = Column('description', TEXT) - create_time = Column('create_time', DateTime, default=now) + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + provider = Column(String(255)) + config = Column(MutableJSONType) + authn_id = Column(Integer, ForeignKey('oidc_user_authnz_tokens.id'), index=True) + tokens = Column(MutableJSONType) + last_update = Column(DateTime) + last_activity = Column(DateTime) + description = Column(TEXT) + create_time = Column(DateTime, default=now) user = relationship('User', back_populates='cloudauthz') authn = relationship('UserAuthnzToken', back_populates='cloudauthz') @@ -7008,17 +7004,17 @@ class Page(Base, Dictifiable, RepresentById): Index('ix_page_slug', 'slug', mysql_length=200), ) - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True, nullable=False) - latest_revision_id = Column('latest_revision_id', Integer, - ForeignKey('page_revision.id', use_alter=True, name='page_latest_revision_id_fk'), index=True) - title = Column('title', TEXT) - deleted = Column('deleted', Boolean, index=True, default=False) - importable = Column('importable', Boolean, index=True, default=False) - slug = Column('slug', TEXT) - published = Column('published', Boolean, index=True, default=False) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=False) + latest_revision_id = Column( + Integer, ForeignKey('page_revision.id', use_alter=True, name='page_latest_revision_id_fk'), index=True) + title = Column(TEXT) + deleted = Column(Boolean, index=True, default=False) + importable = Column(Boolean, index=True, default=False) + slug = Column(TEXT) + published = Column(Boolean, index=True, default=False) user = relationship('User', back_populates='pages') revisions = relationship( 'PageRevision', @@ -7078,13 +7074,13 @@ class Page(Base, Dictifiable, RepresentById): class PageRevision(Base, Dictifiable, RepresentById): __tablename__ = 'page_revision' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - page_id = Column('page_id', Integer, ForeignKey('page.id'), index=True, nullable=False) - title = Column('title', TEXT) - content = Column('content', TEXT) - content_format = Column('content_format', TrimmedString(32)) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + page_id = Column(Integer, ForeignKey('page.id'), index=True, nullable=False) + title = Column(TEXT) + content = Column(TEXT) + content_format = Column(TrimmedString(32)) page = relationship( 'Page', primaryjoin=('Page.id == PageRevision.page_id')) @@ -7104,9 +7100,9 @@ class PageRevision(Base, Dictifiable, RepresentById): class PageUserShareAssociation(Base, UserShareAssociation): __tablename__ = 'page_user_share_association' - id = Column("id", Integer, primary_key=True) - page_id = Column("page_id", Integer, ForeignKey("page.id"), index=True) - user_id = Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True) + id = Column(Integer, primary_key=True) + page_id = Column(Integer, ForeignKey("page.id"), index=True) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) user = relationship('User', back_populates='pages_shared_by_others') page = relationship('Page', back_populates='users_shared_with') @@ -7156,14 +7152,13 @@ class VisualizationRevision(Base, RepresentById): Index('ix_visualization_revision_dbkey', 'dbkey', mysql_length=200), ) - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - update_time = Column('update_time', DateTime, default=now, onupdate=now) - visualization_id = Column('visualization_id', Integer, ForeignKey('visualization.id'), - index=True, nullable=False) - title = Column('title', TEXT) - dbkey = Column('dbkey', TEXT) - config = Column('config', MutableJSONType) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True, nullable=False) + title = Column(TEXT) + dbkey = Column(TEXT) + config = Column(MutableJSONType) def __init__(self, visualization=None, title=None, dbkey=None, config=None): self.id = None @@ -7221,10 +7216,10 @@ class Tag(Base, RepresentById): UniqueConstraint('name'), ) - id = Column('id', Integer, primary_key=True) - type = Column('type', Integer) - parent_id = Column('parent_id', Integer, ForeignKey('tag.id')) - name = Column('name', TrimmedString(255)) + id = Column(Integer, primary_key=True) + type = Column(Integer) + parent_id = Column(Integer, ForeignKey('tag.id')) + name = Column(TrimmedString(255)) children = relationship('Tag', back_populates='parent') parent = relationship('Tag', back_populates='children', remote_side=[id]) tagged_histories = relationship('HistoryTagAssociation', back_populates='tag') @@ -7270,13 +7265,13 @@ class ItemTagAssociation(Dictifiable): class HistoryTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'history_tag_association' - id = Column('id', Integer, primary_key=True) - history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) - tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - user_tname = Column('user_tname', TrimmedString(255), index=True) - value = Column('value', TrimmedString(255), index=True) - user_value = Column('user_value', TrimmedString(255), index=True) + id = Column(Integer, primary_key=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column(TrimmedString(255), index=True) + value = Column(TrimmedString(255), index=True) + user_value = Column(TrimmedString(255), index=True) history = relationship('History', back_populates='tags') tag = relationship('Tag', back_populates='tagged_histories') user = relationship('User') @@ -7285,14 +7280,13 @@ class HistoryTagAssociation(Base, ItemTagAssociation, RepresentById): class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'history_dataset_association_tag_association' - id = Column('id', Integer, primary_key=True) - history_dataset_association_id = Column('history_dataset_association_id', Integer, - ForeignKey('history_dataset_association.id'), index=True) - tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - user_tname = Column('user_tname', TrimmedString(255), index=True) - value = Column('value', TrimmedString(255), index=True) - user_value = Column('user_value', TrimmedString(255), index=True) + id = Column(Integer, primary_key=True) + history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column(TrimmedString(255), index=True) + value = Column(TrimmedString(255), index=True) + user_value = Column(TrimmedString(255), index=True) history_dataset_association = relationship('HistoryDatasetAssociation', back_populates='tags') tag = relationship('Tag', back_populates='tagged_history_dataset_associations') user = relationship('User') @@ -7301,14 +7295,14 @@ class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, Represen class LibraryDatasetDatasetAssociationTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'library_dataset_dataset_association_tag_association' - id = Column('id', Integer, primary_key=True) - library_dataset_dataset_association_id = Column('library_dataset_dataset_association_id', + id = Column(Integer, primary_key=True) + library_dataset_dataset_association_id = Column( Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) - tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - user_tname = Column('user_tname', TrimmedString(255), index=True) - value = Column('value', TrimmedString(255), index=True) - user_value = Column('user_value', TrimmedString(255), index=True) + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column(TrimmedString(255), index=True) + value = Column(TrimmedString(255), index=True) + user_value = Column(TrimmedString(255), index=True) library_dataset_dataset_association = relationship( 'LibraryDatasetDatasetAssociation', back_populates='tags') tag = relationship('Tag', back_populates='tagged_library_dataset_dataset_associations') @@ -7318,13 +7312,13 @@ class LibraryDatasetDatasetAssociationTagAssociation(Base, ItemTagAssociation, R class PageTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'page_tag_association' - id = Column('id', Integer, primary_key=True) - page_id = Column('page_id', Integer, ForeignKey('page.id'), index=True) - tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - user_tname = Column('user_tname', TrimmedString(255), index=True) - value = Column('value', TrimmedString(255), index=True) - user_value = Column('user_value', TrimmedString(255), index=True) + id = Column(Integer, primary_key=True) + page_id = Column(Integer, ForeignKey('page.id'), index=True) + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column(TrimmedString(255), index=True) + value = Column(TrimmedString(255), index=True) + user_value = Column(TrimmedString(255), index=True) page = relationship('Page', back_populates='tags') tag = relationship('Tag', back_populates='tagged_pages') user = relationship('User') @@ -7333,13 +7327,13 @@ class PageTagAssociation(Base, ItemTagAssociation, RepresentById): class WorkflowStepTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'workflow_step_tag_association' - id = Column('id', Integer, primary_key=True) - workflow_step_id = Column('workflow_step_id', Integer, ForeignKey('workflow_step.id'), index=True) - tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - user_tname = Column('user_tname', TrimmedString(255), index=True) - value = Column('value', TrimmedString(255), index=True) - user_value = Column('user_value', TrimmedString(255), index=True) + id = Column(Integer, primary_key=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column(TrimmedString(255), index=True) + value = Column(TrimmedString(255), index=True) + user_value = Column(TrimmedString(255), index=True) workflow_step = relationship('WorkflowStep', back_populates='tags') tag = relationship('Tag', back_populates='tagged_workflow_steps') user = relationship('User') @@ -7348,13 +7342,13 @@ class WorkflowStepTagAssociation(Base, ItemTagAssociation, RepresentById): class StoredWorkflowTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'stored_workflow_tag_association' - id = Column('id', Integer, primary_key=True) - stored_workflow_id = Column('stored_workflow_id', Integer, ForeignKey('stored_workflow.id'), index=True) - tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - user_tname = Column('user_tname', TrimmedString(255), index=True) - value = Column('value', TrimmedString(255), index=True) - user_value = Column('user_value', TrimmedString(255), index=True) + id = Column(Integer, primary_key=True) + stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column(TrimmedString(255), index=True) + value = Column(TrimmedString(255), index=True) + user_value = Column(TrimmedString(255), index=True) stored_workflow = relationship('StoredWorkflow', back_populates='tags') tag = relationship('Tag', back_populates='tagged_stored_workflows') user = relationship('User') @@ -7363,13 +7357,13 @@ class StoredWorkflowTagAssociation(Base, ItemTagAssociation, RepresentById): class VisualizationTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'visualization_tag_association' - id = Column('id', Integer, primary_key=True) - visualization_id = Column('visualization_id', Integer, ForeignKey('visualization.id'), index=True) - tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - user_tname = Column('user_tname', TrimmedString(255), index=True) - value = Column('value', TrimmedString(255), index=True) - user_value = Column('user_value', TrimmedString(255), index=True) + id = Column(Integer, primary_key=True) + visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True) + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column(TrimmedString(255), index=True) + value = Column(TrimmedString(255), index=True) + user_value = Column(TrimmedString(255), index=True) visualization = relationship('Visualization', back_populates='tags') tag = relationship('Tag', back_populates='tagged_visualizations') user = relationship('User') @@ -7378,14 +7372,14 @@ class VisualizationTagAssociation(Base, ItemTagAssociation, RepresentById): class HistoryDatasetCollectionTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'history_dataset_collection_tag_association' - id = Column('id', Integer, primary_key=True) - history_dataset_collection_id = Column('history_dataset_collection_id', Integer, - ForeignKey('history_dataset_collection_association.id'), index=True) - tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - user_tname = Column('user_tname', TrimmedString(255), index=True) - value = Column('value', TrimmedString(255), index=True) - user_value = Column('user_value', TrimmedString(255), index=True) + id = Column(Integer, primary_key=True) + history_dataset_collection_id = Column( + Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column(TrimmedString(255), index=True) + value = Column(TrimmedString(255), index=True) + user_value = Column(TrimmedString(255), index=True) dataset_collection = relationship('HistoryDatasetCollectionAssociation', back_populates='tags') tag = relationship('Tag', back_populates='tagged_history_dataset_collections') user = relationship('User') @@ -7394,14 +7388,14 @@ class HistoryDatasetCollectionTagAssociation(Base, ItemTagAssociation, Represent class LibraryDatasetCollectionTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'library_dataset_collection_tag_association' - id = Column('id', Integer, primary_key=True) - library_dataset_collection_id = Column('library_dataset_collection_id', Integer, - ForeignKey('library_dataset_collection_association.id'), index=True) - tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - user_tname = Column('user_tname', TrimmedString(255), index=True) - value = Column('value', TrimmedString(255), index=True) - user_value = Column('user_value', TrimmedString(255), index=True) + id = Column(Integer, primary_key=True) + library_dataset_collection_id = Column( + Integer, ForeignKey('library_dataset_collection_association.id'), index=True) + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column(TrimmedString(255), index=True) + value = Column(TrimmedString(255), index=True) + user_value = Column(TrimmedString(255), index=True) dataset_collection = relationship('LibraryDatasetCollectionAssociation', back_populates='tags') tag = relationship('Tag', back_populates='tagged_library_dataset_collections') user = relationship('User') @@ -7410,13 +7404,13 @@ class LibraryDatasetCollectionTagAssociation(Base, ItemTagAssociation, Represent class ToolTagAssociation(Base, ItemTagAssociation, RepresentById): __tablename__ = 'tool_tag_association' - id = Column('id', Integer, primary_key=True) - tool_id = Column('tool_id', TrimmedString(255), index=True) - tag_id = Column('tag_id', Integer, ForeignKey('tag.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - user_tname = Column('user_tname', TrimmedString(255), index=True) - value = Column('value', TrimmedString(255), index=True) - user_value = Column('user_value', TrimmedString(255), index=True) + id = Column(Integer, primary_key=True) + tool_id = Column(TrimmedString(255), index=True) + tag_id = Column(Integer, ForeignKey('tag.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user_tname = Column(TrimmedString(255), index=True) + value = Column(TrimmedString(255), index=True) + user_value = Column(TrimmedString(255), index=True) tag = relationship('Tag', back_populates='tagged_tools') user = relationship('User') @@ -7428,10 +7422,10 @@ class HistoryAnnotationAssociation(Base, RepresentById): Index('ix_history_anno_assoc_annotation', 'annotation', mysql_length=200), ) - id = Column('id', Integer, primary_key=True) - history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - annotation = Column('annotation', TEXT) + id = Column(Integer, primary_key=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column(TEXT) history = relationship('History', back_populates='annotations') user = relationship('User') @@ -7442,11 +7436,10 @@ class HistoryDatasetAssociationAnnotationAssociation(Base, RepresentById): Index('ix_history_dataset_anno_assoc_annotation', 'annotation', mysql_length=200), ) - id = Column('id', Integer, primary_key=True) - history_dataset_association_id = Column('history_dataset_association_id', Integer, - ForeignKey('history_dataset_association.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - annotation = Column('annotation', TEXT) + id = Column(Integer, primary_key=True) + history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column(TEXT) hda = relationship('HistoryDatasetAssociation', back_populates='annotations') user = relationship('User') @@ -7457,11 +7450,10 @@ class StoredWorkflowAnnotationAssociation(Base, RepresentById): Index('ix_stored_workflow_ann_assoc_annotation', 'annotation', mysql_length=200), ) - id = Column('id', Integer, primary_key=True) - stored_workflow_id = Column('stored_workflow_id', Integer, - ForeignKey('stored_workflow.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - annotation = Column('annotation', TEXT) + id = Column(Integer, primary_key=True) + stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column(TEXT) stored_workflow = relationship('StoredWorkflow', back_populates='annotations') user = relationship('User') @@ -7472,11 +7464,10 @@ class WorkflowStepAnnotationAssociation(Base, RepresentById): Index('ix_workflow_step_ann_assoc_annotation', 'annotation', mysql_length=200), ) - id = Column('id', Integer, primary_key=True) - workflow_step_id = Column('workflow_step_id', Integer, - ForeignKey('workflow_step.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - annotation = Column('annotation', TEXT) + id = Column(Integer, primary_key=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column(TEXT) workflow_step = relationship('WorkflowStep', back_populates='annotations') user = relationship('User') @@ -7487,10 +7478,10 @@ class PageAnnotationAssociation(Base, RepresentById): Index('ix_page_annotation_association_annotation', 'annotation', mysql_length=200), ) - id = Column('id', Integer, primary_key=True) - page_id = Column('page_id', Integer, ForeignKey('page.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - annotation = Column('annotation', TEXT) + id = Column(Integer, primary_key=True) + page_id = Column(Integer, ForeignKey('page.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column(TEXT) page = relationship('Page', back_populates='annotations') user = relationship('User') @@ -7501,10 +7492,10 @@ class VisualizationAnnotationAssociation(Base, RepresentById): Index('ix_visualization_annotation_association_annotation', 'annotation', mysql_length=200), ) - id = Column('id', Integer, primary_key=True) - visualization_id = Column('visualization_id', Integer, ForeignKey('visualization.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - annotation = Column('annotation', TEXT) + id = Column(Integer, primary_key=True) + visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column(TEXT) visualization = relationship('Visualization', back_populates='annotations') user = relationship('User') @@ -7512,11 +7503,11 @@ class VisualizationAnnotationAssociation(Base, RepresentById): class HistoryDatasetCollectionAssociationAnnotationAssociation(Base, RepresentById): __tablename__ = 'history_dataset_collection_annotation_association' - id = Column('id', Integer, primary_key=True) - history_dataset_collection_id = Column('history_dataset_collection_id', Integer, - ForeignKey('history_dataset_collection_association.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - annotation = Column('annotation', TEXT) + id = Column(Integer, primary_key=True) + history_dataset_collection_id = Column( + Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column(TEXT) history_dataset_collection = relationship('HistoryDatasetCollectionAssociation', back_populates='annotations') user = relationship('User') @@ -7525,11 +7516,11 @@ class HistoryDatasetCollectionAssociationAnnotationAssociation(Base, RepresentBy class LibraryDatasetCollectionAnnotationAssociation(Base, RepresentById): __tablename__ = 'library_dataset_collection_annotation_association' - id = Column('id', Integer, primary_key=True) - library_dataset_collection_id = Column('library_dataset_collection_id', Integer, - ForeignKey('library_dataset_collection_association.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - annotation = Column('annotation', TEXT) + id = Column(Integer, primary_key=True) + library_dataset_collection_id = Column( + Integer, ForeignKey('library_dataset_collection_association.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + annotation = Column(TEXT) dataset_collection = relationship('LibraryDatasetCollectionAssociation', back_populates='annotations') user = relationship('User') @@ -7552,10 +7543,10 @@ class ItemRatingAssociation(Base): class HistoryRatingAssociation(ItemRatingAssociation, RepresentById): __tablename__ = 'history_rating_association' - id = Column('id', Integer, primary_key=True) - history_id = Column('history_id', Integer, ForeignKey('history.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - rating = Column('rating', Integer, index=True) + id = Column(Integer, primary_key=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column(Integer, index=True) history = relationship('History', back_populates='ratings') user = relationship('User') @@ -7566,11 +7557,10 @@ class HistoryRatingAssociation(ItemRatingAssociation, RepresentById): class HistoryDatasetAssociationRatingAssociation(ItemRatingAssociation, RepresentById): __tablename__ = 'history_dataset_association_rating_association' - id = Column('id', Integer, primary_key=True) - history_dataset_association_id = Column('history_dataset_association_id', Integer, - ForeignKey('history_dataset_association.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - rating = Column('rating', Integer, index=True) + id = Column(Integer, primary_key=True) + history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column(Integer, index=True) history_dataset_association = relationship('HistoryDatasetAssociation', back_populates='ratings') user = relationship('User') @@ -7581,10 +7571,10 @@ class HistoryDatasetAssociationRatingAssociation(ItemRatingAssociation, Represen class StoredWorkflowRatingAssociation(ItemRatingAssociation, RepresentById): __tablename__ = 'stored_workflow_rating_association' - id = Column('id', Integer, primary_key=True) - stored_workflow_id = Column('stored_workflow_id', Integer, ForeignKey('stored_workflow.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - rating = Column('rating', Integer, index=True) + id = Column(Integer, primary_key=True) + stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column(Integer, index=True) stored_workflow = relationship('StoredWorkflow', back_populates='ratings') user = relationship('User') @@ -7595,10 +7585,10 @@ class StoredWorkflowRatingAssociation(ItemRatingAssociation, RepresentById): class PageRatingAssociation(ItemRatingAssociation, RepresentById): __tablename__ = 'page_rating_association' - id = Column('id', Integer, primary_key=True) - page_id = Column('page_id', Integer, ForeignKey('page.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - rating = Column('rating', Integer, index=True) + id = Column(Integer, primary_key=True) + page_id = Column(Integer, ForeignKey('page.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column(Integer, index=True) page = relationship('Page', back_populates='ratings') user = relationship('User') @@ -7609,10 +7599,10 @@ class PageRatingAssociation(ItemRatingAssociation, RepresentById): class VisualizationRatingAssociation(ItemRatingAssociation, RepresentById): __tablename__ = 'visualization_rating_association' - id = Column('id', Integer, primary_key=True) - visualization_id = Column('visualization_id', Integer, ForeignKey('visualization.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - rating = Column('rating', Integer, index=True) + id = Column(Integer, primary_key=True) + visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column(Integer, index=True) visualization = relationship('Visualization', back_populates='ratings') user = relationship('User') @@ -7623,11 +7613,11 @@ class VisualizationRatingAssociation(ItemRatingAssociation, RepresentById): class HistoryDatasetCollectionRatingAssociation(ItemRatingAssociation, RepresentById): __tablename__ = 'history_dataset_collection_rating_association' - id = Column('id', Integer, primary_key=True) - history_dataset_collection_id = Column('history_dataset_collection_id', Integer, - ForeignKey('history_dataset_collection_association.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - rating = Column('rating', Integer, index=True) + id = Column(Integer, primary_key=True) + history_dataset_collection_id = Column( + Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column(Integer, index=True) dataset_collection = relationship('HistoryDatasetCollectionAssociation', back_populates='ratings') user = relationship('User') @@ -7638,11 +7628,11 @@ class HistoryDatasetCollectionRatingAssociation(ItemRatingAssociation, Represent class LibraryDatasetCollectionRatingAssociation(ItemRatingAssociation, RepresentById): __tablename__ = 'library_dataset_collection_rating_association' - id = Column('id', Integer, primary_key=True) - library_dataset_collection_id = Column('library_dataset_collection_id', Integer, - ForeignKey('library_dataset_collection_association.id'), index=True) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - rating = Column('rating', Integer, index=True) + id = Column(Integer, primary_key=True) + library_dataset_collection_id = Column( + Integer, ForeignKey('library_dataset_collection_association.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + rating = Column(Integer, index=True) dataset_collection = relationship('LibraryDatasetCollectionAssociation', back_populates='ratings') user = relationship('User') @@ -7674,13 +7664,13 @@ class UserPreference(RepresentById): class UserAction(Base, RepresentById): __tablename__ = 'user_action' - id = Column("id", Integer, primary_key=True) - create_time = Column("create_time", DateTime, default=now) - user_id = Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True) - session_id = Column("session_id", Integer, ForeignKey("galaxy_session.id"), index=True) - action = Column("action", Unicode(255)) - context = Column("context", Unicode(512)) - params = Column("params", Unicode(1024)) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True) + session_id = Column(Integer, ForeignKey("galaxy_session.id"), index=True) + action = Column(Unicode(255)) + context = Column(Unicode(512)) + params = Column(Unicode(1024)) user = relationship('User') def __init__(self, user=None, session_id=None, action=None, params=None, context=None): @@ -7694,10 +7684,10 @@ class UserAction(Base, RepresentById): class APIKeys(Base, RepresentById): __tablename__ = 'api_keys' - id = Column('id', Integer, primary_key=True) - create_time = Column('create_time', DateTime, default=now) - user_id = Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True) - key = Column('key', TrimmedString(32), index=True, unique=True) + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + key = Column(TrimmedString(32), index=True, unique=True) user = relationship('User', back_populates='api_keys') From 3c8d1766d77831310730945f70a02419a3abe8cd Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 22 Jul 2021 20:01:17 -0400 Subject: [PATCH 112/225] Add unit test for History model --- test/unit/model/test_model.py | 147 +++++++++++++++++++++++++++++++++- 1 file changed, 144 insertions(+), 3 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index b19b6c5f477..8c9f0939647 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -531,6 +531,117 @@ class TestGroupRoleAssociation(BaseTest): assert stored_obj.role.id == role.id +class TestHistory(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'history' + # assert has_index(cls_.__table__, ('slug',)) + + def test_columns(self, session, cls_, user): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + name = 'a' + hid_counter = 2 + deleted = True + purged = True + importing = True + genome_build = 'b' + importable = True + slug = 'c' + published = True + obj = cls_(name=name, user=user) + obj.create_time = create_time + obj.update_time = update_time + obj.hid_counter = hid_counter + obj.deleted = deleted + obj.purged = purged + obj.importing = importing + obj.genome_build = genome_build + obj.importable = importable + obj.slug = slug + obj.published = published + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + # cannot test directly (trigger); applies to both column and column property + assert stored_obj.update_time + assert stored_obj.user_id == user.id + assert stored_obj.name == name + assert stored_obj.hid_counter == hid_counter + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + assert stored_obj.importing == importing + assert stored_obj.genome_build == genome_build + assert stored_obj.importable == importable + assert stored_obj.slug == slug + assert stored_obj.published == published + + def test_relationships( + self, + session, + cls_, + user, + history_dataset_association, + job_export_history_archive, + history_dataset_collection_association, + history_tag_association, + history_annotation_association, + history_rating_association, + default_history_permissions, + history_user_share_association, + model, + ): + obj = cls_() + obj.user = user + obj.datasets.append(history_dataset_association) + + # TODO: this is mappend via backref: change to back_populates after mapping HDCA + obj.dataset_collections.append(history_dataset_collection_association) + + obj.exports.append(job_export_history_archive) + obj.tags.append(history_tag_association) + obj.annotations.append(history_annotation_association) + obj.ratings.append(history_rating_association) + obj.default_permissions.append(default_history_permissions) + obj.users_shared_with.append(history_user_share_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id + assert stored_obj.datasets == [history_dataset_association] + assert stored_obj.exports == [job_export_history_archive] + assert stored_obj.active_datasets == [history_dataset_association] + assert stored_obj.active_dataset_collections == [history_dataset_collection_association] + assert stored_obj.visible_datasets == [history_dataset_association] + assert stored_obj.visible_dataset_collections == [history_dataset_collection_association] + assert stored_obj.tags == [history_tag_association] + assert stored_obj.annotations == [history_annotation_association] + assert stored_obj.ratings == [history_rating_association] + # This doesn't test the average amount, just the mapping. + assert stored_obj.average_rating == history_rating_association.rating + assert stored_obj.users_shared_with_count == 1 + assert stored_obj.users_shared_with == [history_user_share_association] + assert stored_obj.default_permissions == [default_history_permissions] + + def test_average_rating(self, model, session, history, user): + # History has been expunged; to access its deferred properties, + # it needs to be added back to the session. + session.add(history) + assert history.average_rating is None # With no ratings, we expect None. + # Create ratings + to_cleanup = [] + for rating in (1, 2, 3, 4, 5): + history_rating_assoc = model.HistoryRatingAssociation(user, history) + history_rating_assoc.rating = rating + persist(session, history_rating_assoc) + to_cleanup.append(history_rating_assoc) + assert history.average_rating == 3.0 # Expect average after ratings added. + # Cleanup: remove ratings from database + delete_from_database(session, to_cleanup) + + class TestHistoryUserShareAssociation(BaseTest): def test_table(self, cls_): @@ -554,8 +665,8 @@ class TestHistoryUserShareAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.history_id == history.id - assert stored_obj.user_id == user.id + assert stored_obj.history.id == history.id + assert stored_obj.user.id == user.id class TestHistoryAudit(BaseTest): @@ -935,7 +1046,7 @@ class TestJobStateHistory(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.job.id == job.id + assert stored_obj.job.id == job.id class TestJobToImplicitOutputDatasetCollectionAssociation(BaseTest): @@ -2756,6 +2867,12 @@ def dataset_source_hash(model, session): yield from dbcleanup_wrapper(session, d) +@pytest.fixture +def default_history_permissions(model, session, history, role): + dha = model.DefaultHistoryPermissions(history, 'a', role) + yield from dbcleanup_wrapper(session, dha) + + @pytest.fixture def default_quota_association(model, session, quota): type_ = model.DefaultQuotaAssociation.types.REGISTERED @@ -2805,6 +2922,12 @@ def history(model, session): yield from dbcleanup_wrapper(session, h) +@pytest.fixture +def history_annotation_association(model, session): + haa = model.HistoryAnnotationAssociation() + yield from dbcleanup_wrapper(session, haa) + + @pytest.fixture def history_dataset_association(model, session, dataset): hda = model.HistoryDatasetAssociation(dataset=dataset) @@ -2829,18 +2952,36 @@ def history_dataset_collection_tag_association(model, session): yield from dbcleanup_wrapper(session, hdcta) +@pytest.fixture +def history_rating_association(model, session, user, history): + hda = model.HistoryRatingAssociation(user, history) + yield from dbcleanup_wrapper(session, hda) + + @pytest.fixture def history_tag_association(model, session): hta = model.HistoryTagAssociation() yield from dbcleanup_wrapper(session, hta) +@pytest.fixture +def history_user_share_association(model, session): + husa = model.HistoryUserShareAssociation() + yield from dbcleanup_wrapper(session, husa) + + @pytest.fixture def job(model, session): j = model.Job() yield from dbcleanup_wrapper(session, j) +@pytest.fixture +def job_export_history_archive(model, session): + jeha = model.JobExportHistoryArchive() + yield from dbcleanup_wrapper(session, jeha) + + @pytest.fixture def library(model, session): lb = model.Library() From 5aa22ad79581151d18f3668748fa2164bf2489b1 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 23 Jul 2021 17:03:36 -0400 Subject: [PATCH 113/225] Map History declaratively (see note) Do not use key attr in column declaration (same as w/LibraryDataset). Use lambda expressions to specify values for order_by and primaryjoin arguments for relationship(). Reasons: more compile-time safety + support for type hints. Slightly less readoable, but, I think, this is a reasoanble tradeoff. TODO: convert all other such instances in the model. --- lib/galaxy/model/__init__.py | 87 +++++++++++++++++++++++++++++++++- lib/galaxy/model/mapping.py | 88 ++--------------------------------- test/unit/model/test_model.py | 6 +-- 3 files changed, 93 insertions(+), 88 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b25742bcdea..e0f6d4fb37a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -39,6 +39,7 @@ from sqlalchemy import ( Boolean, Column, DateTime, + desc, ForeignKey, func, Index, @@ -2126,7 +2127,79 @@ class HistoryAudit(Base, RepresentById): sa_session.execute(d.where(tuple_(history_audit_table.c.history_id, history_audit_table.c.update_time).in_(not_latest_query))) -class History(HasTags, Dictifiable, UsesAnnotations, HasName, RepresentById): +class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, RepresentById): + __tablename__ = 'history' + __table_args__ = ( + Index('ix_history_slug', 'slug', mysql_length=200), + ) + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + _update_time = Column('update_time', DateTime, index=True, default=now, onupdate=now) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + name = Column(TrimmedString(255)) + hid_counter = Column(Integer, default=1) + deleted = Column(Boolean, index=True, default=False) + purged = Column(Boolean, index=True, default=False) + importing = Column(Boolean, index=True, default=False) + genome_build = Column(TrimmedString(40)) + importable = Column(Boolean, default=False) + slug = Column(TEXT) + published = Column(Boolean, index=True, default=False) + + datasets = relationship('HistoryDatasetAssociation', + back_populates='history', + order_by=lambda: asc(HistoryDatasetAssociation.hid)) + exports = relationship('JobExportHistoryArchive', + back_populates='history', + primaryjoin=lambda: JobExportHistoryArchive.history_id == History.id, + order_by=lambda: desc(JobExportHistoryArchive.id)) + active_datasets = relationship('HistoryDatasetAssociation', + primaryjoin=( + lambda: and_(HistoryDatasetAssociation.history_id == History.id, not_(HistoryDatasetAssociation.deleted)) + ), + order_by=lambda: asc(HistoryDatasetAssociation.hid), + viewonly=True) + active_dataset_collections = relationship('HistoryDatasetCollectionAssociation', + primaryjoin=( + lambda: (and_(HistoryDatasetCollectionAssociation.history_id == History.id, + not_(HistoryDatasetCollectionAssociation.deleted))) + ), + order_by=lambda: asc(HistoryDatasetCollectionAssociation.hid), + viewonly=True) + visible_datasets = relationship('HistoryDatasetAssociation', + primaryjoin=( + lambda: and_(HistoryDatasetAssociation.history_id == History.id, + not_(HistoryDatasetAssociation.deleted), HistoryDatasetAssociation.visible) + ), + order_by=lambda: asc(HistoryDatasetAssociation.hid), + viewonly=True) + visible_dataset_collections = relationship('HistoryDatasetCollectionAssociation', + primaryjoin=( + lambda: and_(HistoryDatasetCollectionAssociation.history_id == History.id, + not_(HistoryDatasetCollectionAssociation.deleted), HistoryDatasetCollectionAssociation.visible) + ), + order_by=lambda: asc(HistoryDatasetCollectionAssociation.hid), + viewonly=True) + tags = relationship('HistoryTagAssociation', + order_by=lambda: HistoryTagAssociation.id, + back_populates='history') + annotations = relationship('HistoryAnnotationAssociation', + order_by=lambda: HistoryAnnotationAssociation.id, + back_populates='history') + ratings = relationship('HistoryRatingAssociation', + order_by=lambda: HistoryRatingAssociation.id, + back_populates='history') + default_permissions = relationship('DefaultHistoryPermissions', back_populates='history') + users_shared_with = relationship('HistoryUserShareAssociation', back_populates='history') + + update_time = column_property( + select(func.max(HistoryAudit.update_time)).where(HistoryAudit.history_id == id).scalar_subquery(), + ) + # `users_shared_with_count` and `average_rating` added at the bottom of this module + # (cannot be added before HistoryUserShareAssociation and HistoryRatingAssociation are defined) + users_shared_with_count: column_property + average_rating: column_property dict_collection_visible_keys = ['id', 'name', 'published', 'deleted'] dict_element_visible_keys = ['id', 'name', 'genome_build', 'deleted', 'purged', 'update_time', @@ -7714,8 +7787,18 @@ def _prepare_metadata_for_serialization(id_encoder, serialization_options, metad return processed_metadata -# This must be defined after PageRatingAssociation has been mapped +# Must be defined after PageRatingAssociation has been mapped Page.average_rating = column_property( select(func.avg(PageRatingAssociation.rating)).where(PageRatingAssociation.page_id == Page.id).scalar_subquery(), deferred=True ) +# Must be defined after HistoryUserShareAssociation has been mapped +History.users_shared_with_count = column_property( + select(func.count(HistoryUserShareAssociation.id)).where(History.id == HistoryUserShareAssociation.history_id).scalar_subquery(), + deferred=True +) +# Must be defined after HistoryRatingAssociation has been mapped +History.average_rating = column_property( + select(func.avg(HistoryRatingAssociation.rating)).where(HistoryRatingAssociation.history_id == History.id).scalar_subquery(), + deferred=True +) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6b1a4a2ca17..00c7c500476 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -73,24 +73,6 @@ model.User.table = Table( Column("active", Boolean, index=True, default=True, nullable=False), Column("activation_token", TrimmedString(64), nullable=True, index=True)) -model.History.table = Table( - "history", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, key="_update_time", index=True, default=now, onupdate=now), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("name", TrimmedString(255)), - Column("hid_counter", Integer, default=1), - Column("deleted", Boolean, index=True, default=False), - Column("purged", Boolean, index=True, default=False), - Column("importing", Boolean, index=True, default=False), - Column("genome_build", TrimmedString(40)), - Column("importable", Boolean, default=False), - Column("slug", TEXT), - Column("published", Boolean, index=True, default=False), - Index('ix_history_slug', 'slug', mysql_length=200), -) - model.HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata, Column("id", Integer, primary_key=True), @@ -896,6 +878,7 @@ simple_mapping(model.HistoryDatasetAssociation, dependent_jobs=relation(model.JobToInputDatasetAssociation, back_populates='dataset'), creating_job_associations=relation( model.JobToOutputDatasetAssociation, back_populates='dataset'), + history=relation(model.History, back_populates='datasets'), ) simple_mapping(model.Dataset, @@ -952,68 +935,6 @@ mapper_registry.map_imperatively(model.ImplicitlyConvertedDatasetAssociation, mo backref="implicitly_converted_parent_datasets") )) -mapper_registry.map_imperatively(model.History, model.History.table, properties=dict( - datasets=relation(model.HistoryDatasetAssociation, - backref="history", - order_by=asc(model.HistoryDatasetAssociation.table.c.hid)), - exports=relation(model.JobExportHistoryArchive, - backref="history", - primaryjoin=(model.JobExportHistoryArchive.table.c.history_id == model.History.table.c.id), - order_by=desc(model.JobExportHistoryArchive.table.c.id)), - active_datasets=relation(model.HistoryDatasetAssociation, - primaryjoin=( - (model.HistoryDatasetAssociation.table.c.history_id == model.History.table.c.id) - & not_(model.HistoryDatasetAssociation.table.c.deleted) - ), - order_by=asc(model.HistoryDatasetAssociation.table.c.hid), - viewonly=True), - active_dataset_collections=relation(model.HistoryDatasetCollectionAssociation, - primaryjoin=( - (model.HistoryDatasetCollectionAssociation.table.c.history_id == model.History.table.c.id) - & not_(model.HistoryDatasetCollectionAssociation.table.c.deleted) - ), - order_by=asc(model.HistoryDatasetCollectionAssociation.table.c.hid), - viewonly=True), - visible_datasets=relation(model.HistoryDatasetAssociation, - primaryjoin=( - (model.HistoryDatasetAssociation.table.c.history_id == model.History.table.c.id) - & not_(model.HistoryDatasetAssociation.table.c.deleted) - & model.HistoryDatasetAssociation.table.c.visible - ), - order_by=asc(model.HistoryDatasetAssociation.table.c.hid), - viewonly=True), - visible_dataset_collections=relation(model.HistoryDatasetCollectionAssociation, - primaryjoin=( - (model.HistoryDatasetCollectionAssociation.table.c.history_id == model.History.table.c.id) - & not_(model.HistoryDatasetCollectionAssociation.table.c.deleted) - & model.HistoryDatasetCollectionAssociation.table.c.visible - ), - order_by=asc(model.HistoryDatasetCollectionAssociation.table.c.hid), - viewonly=True), - tags=relation(model.HistoryTagAssociation, - order_by=model.HistoryTagAssociation.id, - back_populates="history"), - annotations=relation(model.HistoryAnnotationAssociation, - order_by=model.HistoryAnnotationAssociation.id, - back_populates="history"), - ratings=relation(model.HistoryRatingAssociation, - order_by=model.HistoryRatingAssociation.id, - back_populates="history"), - average_rating=column_property( - select(func.avg(model.HistoryRatingAssociation.rating)).where(model.HistoryRatingAssociation.history_id == model.History.table.c.id).scalar_subquery(), - deferred=True - ), - users_shared_with_count=column_property( - select(func.count(model.HistoryUserShareAssociation.id)).where(model.History.table.c.id == model.HistoryUserShareAssociation.history_id).scalar_subquery(), - deferred=True - ), - update_time=column_property( - select(func.max(model.HistoryAudit.update_time)).where(model.HistoryAudit.history_id == model.History.table.c.id).scalar_subquery(), - ), - default_permissions=relation(model.DefaultHistoryPermissions, back_populates='history'), - users_shared_with=relation(model.HistoryUserShareAssociation, back_populates='history'), -)) - # Set up proxy so that # History.users_shared_with # returns a list of users that history is shared with. @@ -1032,8 +953,8 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( order_by=desc(model.History.update_time)), active_histories=relation(model.History, primaryjoin=( - (model.History.table.c.user_id == model.User.table.c.id) - & (not_(model.History.table.c.deleted)) + (model.History.user_id == model.User.table.c.id) + & (not_(model.History.deleted)) ), viewonly=True, order_by=desc(model.History.update_time)), @@ -1208,7 +1129,8 @@ mapper_registry.map_imperatively(model.JobExternalOutputMetadata, model.JobExter mapper_registry.map_imperatively(model.JobExportHistoryArchive, model.JobExportHistoryArchive.table, properties=dict( job=relation(model.Job), - dataset=relation(model.Dataset, backref='job_export_history_archive') + dataset=relation(model.Dataset, backref='job_export_history_archive'), + history=relation(model.History, back_populates='exports'), )) mapper_registry.map_imperatively(model.JobImportHistoryArchive, model.JobImportHistoryArchive.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 8c9f0939647..ddb5b8e1cb8 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -533,9 +533,9 @@ class TestGroupRoleAssociation(BaseTest): class TestHistory(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'history' - # assert has_index(cls_.__table__, ('slug',)) + def test_table(self, cls_): + assert cls_.__tablename__ == 'history' + assert has_index(cls_.__table__, ('slug',)) def test_columns(self, session, cls_, user): create_time = datetime.now() From ba037e470ff33d645f6099f70739e25a94ed99dc Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 23 Jul 2021 17:43:16 -0400 Subject: [PATCH 114/225] Do not type-check the body of lambda expressions Because this would be done prior to mapper-resolution stage, and types could not be determined, causing mypy errors. --- lib/galaxy/model/__init__.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e0f6d4fb37a..71755bfc19d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2149,46 +2149,46 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, RepresentByI datasets = relationship('HistoryDatasetAssociation', back_populates='history', - order_by=lambda: asc(HistoryDatasetAssociation.hid)) + order_by=lambda: asc(HistoryDatasetAssociation.hid)) # type: ignore exports = relationship('JobExportHistoryArchive', back_populates='history', - primaryjoin=lambda: JobExportHistoryArchive.history_id == History.id, - order_by=lambda: desc(JobExportHistoryArchive.id)) + primaryjoin=lambda: JobExportHistoryArchive.history_id == History.id, # type: ignore + order_by=lambda: desc(JobExportHistoryArchive.id)) # type: ignore active_datasets = relationship('HistoryDatasetAssociation', primaryjoin=( - lambda: and_(HistoryDatasetAssociation.history_id == History.id, not_(HistoryDatasetAssociation.deleted)) + lambda: and_(HistoryDatasetAssociation.history_id == History.id, not_(HistoryDatasetAssociation.deleted)) # type: ignore ), - order_by=lambda: asc(HistoryDatasetAssociation.hid), + order_by=lambda: asc(HistoryDatasetAssociation.hid), # type: ignore viewonly=True) active_dataset_collections = relationship('HistoryDatasetCollectionAssociation', primaryjoin=( - lambda: (and_(HistoryDatasetCollectionAssociation.history_id == History.id, - not_(HistoryDatasetCollectionAssociation.deleted))) + lambda: (and_(HistoryDatasetCollectionAssociation.history_id == History.id, # type: ignore + not_(HistoryDatasetCollectionAssociation.deleted))) # type: ignore ), - order_by=lambda: asc(HistoryDatasetCollectionAssociation.hid), + order_by=lambda: asc(HistoryDatasetCollectionAssociation.hid), # type: ignore viewonly=True) visible_datasets = relationship('HistoryDatasetAssociation', primaryjoin=( - lambda: and_(HistoryDatasetAssociation.history_id == History.id, - not_(HistoryDatasetAssociation.deleted), HistoryDatasetAssociation.visible) + lambda: and_(HistoryDatasetAssociation.history_id == History.id, # type: ignore + not_(HistoryDatasetAssociation.deleted), HistoryDatasetAssociation.visible) # type: ignore ), - order_by=lambda: asc(HistoryDatasetAssociation.hid), + order_by=lambda: asc(HistoryDatasetAssociation.hid), # type: ignore viewonly=True) visible_dataset_collections = relationship('HistoryDatasetCollectionAssociation', primaryjoin=( - lambda: and_(HistoryDatasetCollectionAssociation.history_id == History.id, - not_(HistoryDatasetCollectionAssociation.deleted), HistoryDatasetCollectionAssociation.visible) + lambda: and_(HistoryDatasetCollectionAssociation.history_id == History.id, # type: ignore + not_(HistoryDatasetCollectionAssociation.deleted), HistoryDatasetCollectionAssociation.visible) # type: ignore ), - order_by=lambda: asc(HistoryDatasetCollectionAssociation.hid), + order_by=lambda: asc(HistoryDatasetCollectionAssociation.hid), # type: ignore viewonly=True) tags = relationship('HistoryTagAssociation', - order_by=lambda: HistoryTagAssociation.id, + order_by=lambda: HistoryTagAssociation.id, # type: ignore back_populates='history') annotations = relationship('HistoryAnnotationAssociation', - order_by=lambda: HistoryAnnotationAssociation.id, + order_by=lambda: HistoryAnnotationAssociation.id, # type: ignore back_populates='history') ratings = relationship('HistoryRatingAssociation', - order_by=lambda: HistoryRatingAssociation.id, + order_by=lambda: HistoryRatingAssociation.id, # type: ignore back_populates='history') default_permissions = relationship('DefaultHistoryPermissions', back_populates='history') users_shared_with = relationship('HistoryUserShareAssociation', back_populates='history') From 391514cf3a59a308c9f7194e7090ed70a50989f7 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 23 Jul 2021 18:29:58 -0400 Subject: [PATCH 115/225] Add test for Dataset model --- test/unit/model/test_model.py | 92 +++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index ddb5b8e1cb8..a68f8abf47c 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -125,6 +125,92 @@ class TestCloudAuthz(BaseTest): assert stored_obj.authn.id == user_authnz_token.id +class TestDataset(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'dataset' + + def test_columns(self, session, cls_, job): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + state = 'a' + deleted = True + purged = True + purgable = False + object_store_id = 'b' + external_filename = 'c' + _extra_files_path = 'd' + created_from_basename = 'e' + file_size = 1 + total_size = 2 + + obj = cls_() + obj.job = job + obj.create_time = create_time + obj.update_time = update_time + obj.state = state + obj.deleted = deleted + obj.purged = purged + obj.purgable = purgable + obj.object_store_id = object_store_id + obj.external_filename = external_filename + obj._extra_files_path = _extra_files_path + obj.created_from_basename = created_from_basename + obj.file_size = file_size + obj.total_size = total_size + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.state == state + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + assert stored_obj.purgable == purgable + assert stored_obj.object_store_id == object_store_id + assert stored_obj.external_filename == external_filename + assert stored_obj._extra_files_path == _extra_files_path + assert stored_obj.created_from_basename == created_from_basename + assert stored_obj.file_size == file_size + assert stored_obj.total_size == total_size + + def test_relationships( + self, + session, + cls_, + job, + dataset_permission, + history_dataset_association, + library_dataset_dataset_association, + dataset_hash, + dataset_source, + model, + ): + obj = cls_() + obj.job = job + obj.actions.append(dataset_permission) + obj.history_associations.append(history_dataset_association) # TODO: mappend via backref, map explicitly after HDA is mapped + obj.library_associations.append(library_dataset_dataset_association) # TODO: mappend via backref, map explicitly after LDDA is mapped + obj.hashes.append(dataset_hash) + obj.sources.append(dataset_source) + + hda = model.HistoryDatasetAssociation() + hda.purged = True + obj.history_associations.append(hda) + + with dbcleanup(session, obj) as obj_id, dbcleanup(session, hda): + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.actions == [dataset_permission] + assert stored_obj.active_history_associations == [history_dataset_association] + assert stored_obj.purged_history_associations[0].id == hda.id + assert stored_obj.active_library_associations == [library_dataset_dataset_association] + assert stored_obj.hashes == [dataset_hash] + assert stored_obj.sources == [dataset_source] + + class TestDatasetHash(BaseTest): def test_table(self, cls_): @@ -2849,6 +2935,12 @@ def dataset_collection_element( yield from dbcleanup_wrapper(session, dce) +@pytest.fixture +def dataset_hash(model, session): + dh = model.DatasetHash() + yield from dbcleanup_wrapper(session, dh) + + @pytest.fixture def dataset_permission(model, session, dataset): d = model.DatasetPermissions('a', dataset) From f79b1efe662fcde25005ed77d44813e6db2d30ce Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 12:07:50 -0400 Subject: [PATCH 116/225] Add unit test for Event model --- test/unit/model/test_model.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index a68f8abf47c..82b65875a4b 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -456,6 +456,41 @@ class TestDynamicTool(BaseTest): assert stored_obj.workflow_steps == [workflow_step] +class TestEvent(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'event' + + def test_columns(self, session, cls_, history, galaxy_session, user): + message, tool_id = 'a', 'b' + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj = cls_(message, history, user, galaxy_session) + obj.create_time = create_time + obj.update_time = update_time + obj.tool_id = tool_id + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id + assert stored_obj.message == message + assert stored_obj.session_id == galaxy_session.id + assert stored_obj.tool_id == tool_id + + def test_relationships(self, session, cls_, history, galaxy_session, user): + obj = cls_(None, history, user, galaxy_session) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history.id == history.id + assert stored_obj.user.id == user.id + assert stored_obj.galaxy_session.id == galaxy_session.id + + class TestFormDefinition(BaseTest): def test_table(self, cls_): From 5df772f6d2761cc003598c892203b29cf861ae4c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 12:14:25 -0400 Subject: [PATCH 117/225] Map Event declaratively --- lib/galaxy/model/__init__.py | 17 ++++++++++++++++- lib/galaxy/model/mapping.py | 18 ------------------ test/unit/model/test_model.py | 4 ++-- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 71755bfc19d..06b9fda63aa 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5484,7 +5484,22 @@ class DatasetCollectionElement(Dictifiable, RepresentById): return rval -class Event(RepresentById): +class Event(Base, RepresentById): + __tablename__ = 'event' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + history_id = Column(Integer, ForeignKey('history.id'), index=True, nullable=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=True) + message = Column(TrimmedString(1024)) + session_id = Column(Integer, ForeignKey('galaxy_session.id'), index=True, nullable=True) + tool_id = Column(String(255)) + + history = relationship('History') + user = relationship('User') + galaxy_session = relationship('GalaxySession') + def __init__(self, message=None, history=None, user=None, galaxy_session=None): self.history = history self.galaxy_session = galaxy_session diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 00c7c500476..98f54ef412c 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -463,17 +463,6 @@ model.DatasetCollectionElement.table = Table( Column("element_index", Integer), Column("element_identifier", Unicode(255), )) -model.Event.table = Table( - "event", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("history_id", Integer, ForeignKey("history.id"), index=True, nullable=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True, nullable=True), - Column("message", TrimmedString(1024)), - Column("session_id", Integer, ForeignKey("galaxy_session.id"), index=True, nullable=True), - Column("tool_id", String(255))) - model.GalaxySession.table = Table( "galaxy_session", metadata, Column("id", Integer, primary_key=True), @@ -1245,13 +1234,6 @@ simple_mapping(model.DatasetCollectionElement, child_collection=relation(model.DatasetCollection, primaryjoin=(model.DatasetCollectionElement.table.c.child_collection_id == model.DatasetCollection.table.c.id))) -mapper_registry.map_imperatively(model.Event, model.Event.table, properties=dict( - history=relation(model.History), - galaxy_session=relation(model.GalaxySession), - # user=relation( model.User.mapper ) ) ) - user=relation(model.User) -)) - mapper_registry.map_imperatively(model.GalaxySession, model.GalaxySession.table, properties=dict( current_history=relation(model.History), )) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 82b65875a4b..3d427bc30ca 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -458,8 +458,8 @@ class TestDynamicTool(BaseTest): class TestEvent(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'event' + def test_table(self, cls_): + assert cls_.__tablename__ == 'event' def test_columns(self, session, cls_, history, galaxy_session, user): message, tool_id = 'a', 'b' From 04807e5fac9ca55c65a7a714282c2f6d41a7e7bf Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 12:37:19 -0400 Subject: [PATCH 118/225] Add unit test for GalaxySession model --- test/unit/model/test_model.py | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 3d427bc30ca..e1f8eaa7753 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -594,6 +594,60 @@ class TestFormValues(BaseTest): assert stored_obj.form_definition.id == form_definition.id +class TestGalaxySession(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'galaxy_session' + + def test_columns(self, session, cls_, user, history, galaxy_session): + + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + remote_host = 'a' + remote_addr = 'b' + referer = 'c' + session_key = 'd' + is_valid = True + disk_usage = 9 + last_action = update_time + timedelta(hours=1) + + obj = cls_(user=user, current_history=history, prev_session_id=galaxy_session.id) + + obj.create_time = create_time + obj.update_time = update_time + obj.remote_host = remote_host + obj.remote_addr = remote_addr + obj.referer = referer + obj.session_key = session_key + obj.is_valid = is_valid + obj.disk_usage = disk_usage + obj.last_action = last_action + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.user_id == user.id + assert stored_obj.remote_host == remote_host + assert stored_obj.remote_addr == remote_addr + assert stored_obj.referer == referer + assert stored_obj.current_history_id == history.id + assert stored_obj.session_key == session_key + assert stored_obj.is_valid == is_valid + assert stored_obj.prev_session_id == galaxy_session.id + assert stored_obj.disk_usage == disk_usage + assert stored_obj.last_action == last_action + + def test_relationships(self, session, cls_, user, history): + obj = cls_(user=user, current_history=history) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id + assert stored_obj.current_history.id == history.id + + class TestGroupQuotaAssociation(BaseTest): def test_table(self, cls_): From ebefb7b7aa381cd400cef77237f4ce9b99bb8799 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 12:56:11 -0400 Subject: [PATCH 119/225] Map GalaxySession declaratively --- lib/galaxy/model/__init__.py | 21 ++++++++++++++++++++- lib/galaxy/model/mapping.py | 24 +----------------------- test/unit/model/test_model.py | 4 ++-- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 06b9fda63aa..4532e6b2219 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5508,7 +5508,26 @@ class Event(Base, RepresentById): self.message = message -class GalaxySession(RepresentById): +class GalaxySession(Base, RepresentById): + __tablename__ = 'galaxy_session' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=True) + remote_host = Column(String(255)) + remote_addr = Column(String(255)) + referer = Column(TEXT) + current_history_id = Column(Integer, ForeignKey('history.id'), nullable=True) + # unique 128 bit random number coerced to a string + session_key = Column(TrimmedString(255), index=True, unique=True) + is_valid = Column(Boolean, default=False) + # saves a reference to the previous session so we have a way to chain them together + prev_session_id = Column(Integer) + disk_usage = Column(Numeric(15, 0), index=True) + last_action = Column(DateTime) + current_history = relationship('History') + def __init__(self, id=None, user=None, diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 98f54ef412c..0ed756bf90f 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -463,24 +463,6 @@ model.DatasetCollectionElement.table = Table( Column("element_index", Integer), Column("element_identifier", Unicode(255), )) -model.GalaxySession.table = Table( - "galaxy_session", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True, nullable=True), - Column("remote_host", String(255)), - Column("remote_addr", String(255)), - Column("referer", TEXT), - Column("current_history_id", Integer, ForeignKey("history.id"), nullable=True), - # unique 128 bit random number coerced to a string - Column("session_key", TrimmedString(255), index=True, unique=True), - Column("is_valid", Boolean, default=False), - # saves a reference to the previous session so we have a way to chain them together - Column("prev_session_id", Integer), - Column("disk_usage", Numeric(15, 0), index=True), - Column("last_action", DateTime)) - model.GalaxySessionToHistoryAssociation.table = Table( "galaxy_session_to_history", metadata, Column("id", Integer, primary_key=True), @@ -949,7 +931,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( order_by=desc(model.History.update_time)), galaxy_sessions=relation(model.GalaxySession, backref="user", - order_by=desc(model.GalaxySession.table.c.update_time)), + order_by=desc(model.GalaxySession.update_time)), pages_shared_by_others=relation(model.PageUserShareAssociation, back_populates='user'), quotas=relation(model.UserQuotaAssociation, back_populates='user'), social_auth=relation(model.UserAuthnzToken, back_populates='user'), @@ -1234,10 +1216,6 @@ simple_mapping(model.DatasetCollectionElement, child_collection=relation(model.DatasetCollection, primaryjoin=(model.DatasetCollectionElement.table.c.child_collection_id == model.DatasetCollection.table.c.id))) -mapper_registry.map_imperatively(model.GalaxySession, model.GalaxySession.table, properties=dict( - current_history=relation(model.History), -)) - mapper_registry.map_imperatively(model.GalaxySessionToHistoryAssociation, model.GalaxySessionToHistoryAssociation.table, properties=dict( galaxy_session=relation(model.GalaxySession, backref='histories'), history=relation(model.History, backref='galaxy_sessions') diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index e1f8eaa7753..6d674afcff0 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -596,8 +596,8 @@ class TestFormValues(BaseTest): class TestGalaxySession(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'galaxy_session' + def test_table(self, cls_): + assert cls_.__tablename__ == 'galaxy_session' def test_columns(self, session, cls_, user, history, galaxy_session): From 1c6db5aa0d739e85e65ee440f583e30adee46f2e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 13:27:36 -0400 Subject: [PATCH 120/225] Map GalaxySessionToHistoryAssoc declaratively --- lib/galaxy/model/__init__.py | 13 +++++++++++- lib/galaxy/model/mapping.py | 12 ----------- test/unit/model/test_model.py | 39 ++++++++++++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4532e6b2219..3155cec4b17 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2192,6 +2192,7 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, RepresentByI back_populates='history') default_permissions = relationship('DefaultHistoryPermissions', back_populates='history') users_shared_with = relationship('HistoryUserShareAssociation', back_populates='history') + galaxy_sessions = relationship('GalaxySessionToHistoryAssociation', back_populates='history') update_time = column_property( select(func.max(HistoryAudit.update_time)).where(HistoryAudit.history_id == id).scalar_subquery(), @@ -5527,6 +5528,7 @@ class GalaxySession(Base, RepresentById): disk_usage = Column(Numeric(15, 0), index=True) last_action = Column(DateTime) current_history = relationship('History') + histories = relationship('GalaxySessionToHistoryAssociation', back_populates='galaxy_session') def __init__(self, id=None, @@ -5567,7 +5569,16 @@ class GalaxySession(Base, RepresentById): total_disk_usage = property(get_disk_usage, set_disk_usage) -class GalaxySessionToHistoryAssociation(RepresentById): +class GalaxySessionToHistoryAssociation(Base, RepresentById): + __tablename__ = 'galaxy_session_to_history' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + session_id = Column(Integer, ForeignKey('galaxy_session.id'), index=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + galaxy_session = relationship('GalaxySession', back_populates='histories') + history = relationship('History', back_populates='galaxy_sessions') + def __init__(self, galaxy_session, history): self.galaxy_session = galaxy_session self.history = history diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 0ed756bf90f..1bb0137b18b 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -463,13 +463,6 @@ model.DatasetCollectionElement.table = Table( Column("element_index", Integer), Column("element_identifier", Unicode(255), )) -model.GalaxySessionToHistoryAssociation.table = Table( - "galaxy_session_to_history", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("session_id", Integer, ForeignKey("galaxy_session.id"), index=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True)) - model.StoredWorkflow.table = Table( "stored_workflow", metadata, Column("id", Integer, primary_key=True), @@ -1216,11 +1209,6 @@ simple_mapping(model.DatasetCollectionElement, child_collection=relation(model.DatasetCollection, primaryjoin=(model.DatasetCollectionElement.table.c.child_collection_id == model.DatasetCollection.table.c.id))) -mapper_registry.map_imperatively(model.GalaxySessionToHistoryAssociation, model.GalaxySessionToHistoryAssociation.table, properties=dict( - galaxy_session=relation(model.GalaxySession, backref='histories'), - history=relation(model.History, backref='galaxy_sessions') -)) - mapper_registry.map_imperatively(model.Workflow, model.Workflow.table, properties=dict( steps=relation(model.WorkflowStep, backref='workflow', diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 6d674afcff0..53b954fe125 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -639,13 +639,41 @@ class TestGalaxySession(BaseTest): assert stored_obj.disk_usage == disk_usage assert stored_obj.last_action == last_action - def test_relationships(self, session, cls_, user, history): + def test_relationships(self, session, cls_, user, history, galaxy_session_history_association): obj = cls_(user=user, current_history=history) + obj.histories.append(galaxy_session_history_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.user.id == user.id assert stored_obj.current_history.id == history.id + assert stored_obj.histories == [galaxy_session_history_association] + + +class TestGalaxySessionToHistoryAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'galaxy_session_to_history' + + def test_columns(self, session, cls_, galaxy_session, history): + create_time = datetime.now() + obj = cls_(galaxy_session, history) + obj.create_time = create_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.session_id == galaxy_session.id + assert stored_obj.history_id == history.id + + def test_relationships(self, session, cls_, galaxy_session, history): + obj = cls_(galaxy_session, history) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.galaxy_session.id == galaxy_session.id + assert stored_obj.history.id == history.id class TestGroupQuotaAssociation(BaseTest): @@ -766,6 +794,7 @@ class TestHistory(BaseTest): history_rating_association, default_history_permissions, history_user_share_association, + galaxy_session_history_association, model, ): obj = cls_() @@ -781,6 +810,7 @@ class TestHistory(BaseTest): obj.ratings.append(history_rating_association) obj.default_permissions.append(default_history_permissions) obj.users_shared_with.append(history_user_share_association) + obj.galaxy_sessions.append(galaxy_session_history_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -799,6 +829,7 @@ class TestHistory(BaseTest): assert stored_obj.users_shared_with_count == 1 assert stored_obj.users_shared_with == [history_user_share_association] assert stored_obj.default_permissions == [default_history_permissions] + assert stored_obj.galaxy_sessions == [galaxy_session_history_association] def test_average_rating(self, model, session, history, user): # History has been expunged; to access its deferred properties, @@ -3079,6 +3110,12 @@ def galaxy_session(model, session, user): yield from dbcleanup_wrapper(session, s) +@pytest.fixture +def galaxy_session_history_association(model, session, galaxy_session, history): + sha = model.GalaxySessionToHistoryAssociation(galaxy_session, history) + yield from dbcleanup_wrapper(session, sha) + + @pytest.fixture def group(model, session): g = model.Group() From fd92cc3f98653bd8c73cb3de4ab22fa627517471 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 14:49:20 -0400 Subject: [PATCH 121/225] Map HDAHistory declaratively --- lib/galaxy/model/__init__.py | 14 +++++++++++++- lib/galaxy/model/mapping.py | 15 --------------- test/unit/model/test_model.py | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3155cec4b17..f0c551f805d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -83,6 +83,7 @@ import galaxy.model.tags import galaxy.security.passwords import galaxy.util from galaxy.model.custom_types import ( + JSONType, MutableJSONType, TrimmedString, UUIDType, @@ -4127,7 +4128,18 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot + type_coerce(cls.id, Unicode)).label('type_id')) -class HistoryDatasetAssociationHistory(RepresentById): +class HistoryDatasetAssociationHistory(Base, RepresentById): + __tablename__ = 'history_dataset_association_history' + + id = Column(Integer, primary_key=True) + history_dataset_association_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + update_time = Column(DateTime, default=now) + version = Column(Integer) + name = Column(TrimmedString(255)) + extension = Column(TrimmedString(64)) + _metadata = Column('metadata', JSONType) + extended_metadata_id = Column(Integer, ForeignKey("extended_metadata.id"), index=True) + def __init__(self, history_dataset_association_id, name, diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 1bb0137b18b..c26ed2ea2dc 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -106,19 +106,6 @@ model.HistoryDatasetAssociation.table = Table( ForeignKey("history_dataset_collection_association.id"), nullable=True)) -model.HistoryDatasetAssociationHistory.table = Table( - "history_dataset_association_history", metadata, - Column("id", Integer, primary_key=True), - Column("history_dataset_association_id", Integer, ForeignKey("history_dataset_association.id"), index=True), - Column("update_time", DateTime, default=now), - Column("version", Integer), - Column("name", TrimmedString(255)), - Column("extension", TrimmedString(64)), - Column("metadata", JSONType, key="_metadata"), - Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True), -) - - model.Dataset.table = Table( "dataset", metadata, Column("id", Integer, primary_key=True), @@ -868,8 +855,6 @@ simple_mapping(model.Dataset, sources=relation(model.DatasetSource, back_populates='dataset'), ) -mapper_registry.map_imperatively(model.HistoryDatasetAssociationHistory, model.HistoryDatasetAssociationHistory.table) - mapper_registry.map_imperatively(model.HistoryDatasetAssociationDisplayAtAuthorization, model.HistoryDatasetAssociationDisplayAtAuthorization.table, properties=dict( history_dataset_association=relation(model.HistoryDatasetAssociation), user=relation(model.User) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 53b954fe125..467b2288e53 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -848,6 +848,36 @@ class TestHistory(BaseTest): delete_from_database(session, to_cleanup) +class TestHistoryDatasetAssociationHistory(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_association_history' + + def test_columns(self, session, cls_, history_dataset_association, extended_metadata): + name, update_time, version, extension, metadata = 'a', datetime.now(), 2, 'b', 'c' + obj = cls_( + history_dataset_association.id, + name, + None, + update_time, + version, + extension, + extended_metadata.id, + metadata + ) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.name == name + assert stored_obj.update_time == update_time + assert stored_obj.version == version + assert stored_obj.extension == extension + assert stored_obj.extended_metadata_id == extended_metadata.id + assert stored_obj._metadata == metadata + + class TestHistoryUserShareAssociation(BaseTest): def test_table(self, cls_): @@ -3092,6 +3122,12 @@ def default_quota_association(model, session, quota): yield from dbcleanup_wrapper(session, dqa) +@pytest.fixture +def extended_metadata(model, session): + em = model.ExtendedMetadata(None) + yield from dbcleanup_wrapper(session, em) + + @pytest.fixture def form_definition(model, session, form_definition_current): fd = model.FormDefinition(name='a', form_definition_current=form_definition_current) From 8869c8ae64aebbebe050e6da244227e1c83e176b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 17:36:40 -0400 Subject: [PATCH 122/225] Map ExtendedMetadata declaratively --- lib/galaxy/model/__init__.py | 8 +++++++- lib/galaxy/model/mapping.py | 15 ++++----------- test/unit/model/test_model.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index f0c551f805d..0438fa5c3e0 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4673,7 +4673,13 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): log.warning(f'Attempt to updated parent folder times failed: {ret.rowcount} records updated.') -class ExtendedMetadata(RepresentById): +class ExtendedMetadata(Base, RepresentById): + __tablename__ = 'extended_metadata' + + id = Column(Integer, primary_key=True) + data = Column(MutableJSONType) + children = relationship('ExtendedMetadataIndex', back_populates='extended_metadata') + def __init__(self, data): self.data = data diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index c26ed2ea2dc..18314a19f5d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -195,11 +195,6 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), Column("message", TrimmedString(255))) -model.ExtendedMetadata.table = Table( - "extended_metadata", metadata, - Column("id", Integer, primary_key=True), - Column("data", MutableJSONType)) - model.ExtendedMetadataIndex.table = Table( "extended_metadata_index", metadata, Column("id", Integer, primary_key=True), @@ -819,7 +814,7 @@ simple_mapping(model.HistoryDatasetAssociation, back_populates="history_dataset_association"), extended_metadata=relation(model.ExtendedMetadata, primaryjoin=(model.HistoryDatasetAssociation.table.c.extended_metadata_id - == model.ExtendedMetadata.table.c.id)), + == model.ExtendedMetadata.id)), hidden_beneath_collection_instance=relation(model.HistoryDatasetCollectionAssociation, primaryjoin=(model.HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id == model.HistoryDatasetCollectionAssociation.table.c.id), @@ -954,12 +949,10 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.ExtendedMetadata, model.ExtendedMetadata.table, properties=dict( - children=relation(model.ExtendedMetadataIndex, backref='extended_metadata') +mapper_registry.map_imperatively(model.ExtendedMetadataIndex, model.ExtendedMetadataIndex.table, properties=dict( + extended_metadata=relation(model.ExtendedMetadata, back_populates='children') )) -mapper_registry.map_imperatively(model.ExtendedMetadataIndex, model.ExtendedMetadataIndex.table) - mapper_registry.map_imperatively(model.LibraryInfoAssociation, model.LibraryInfoAssociation.table, properties=dict( library=relation(model.Library, primaryjoin=( @@ -1012,7 +1005,7 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.L order_by=model.LibraryDatasetDatasetAssociationTagAssociation.id, back_populates='library_dataset_dataset_association'), extended_metadata=relation(model.ExtendedMetadata, - primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.extended_metadata_id == model.ExtendedMetadata.table.c.id) + primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.extended_metadata_id == model.ExtendedMetadata.id) ), _metadata=deferred(model.LibraryDatasetDatasetAssociation.table.c._metadata), actions=relation( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 467b2288e53..4b3706d24ba 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -491,6 +491,29 @@ class TestEvent(BaseTest): assert stored_obj.galaxy_session.id == galaxy_session.id +class TestExtendedMetadata(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'extended_metadata' + + def test_columns(self, session, cls_): + data = 'a' + obj = cls_(data) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.data == data + + def test_relationships(self, session, cls_, extended_metadata_index): + obj = cls_(None) + obj.children.append(extended_metadata_index) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.children == [extended_metadata_index] + + class TestFormDefinition(BaseTest): def test_table(self, cls_): @@ -3128,6 +3151,12 @@ def extended_metadata(model, session): yield from dbcleanup_wrapper(session, em) +@pytest.fixture +def extended_metadata_index(model, session, extended_metadata): + emi = model.ExtendedMetadataIndex(extended_metadata, None, None) + yield from dbcleanup_wrapper(session, emi) + + @pytest.fixture def form_definition(model, session, form_definition_current): fd = model.FormDefinition(name='a', form_definition_current=form_definition_current) From c50f0be1fa9649800c29e0dd37fcfd049120a5ed Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 17:44:26 -0400 Subject: [PATCH 123/225] Map ExtendedMetadataIndex declaratively --- lib/galaxy/model/__init__.py | 11 ++++++++++- lib/galaxy/model/mapping.py | 12 ------------ test/unit/model/test_model.py | 24 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 0438fa5c3e0..b166ef28e50 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4684,7 +4684,16 @@ class ExtendedMetadata(Base, RepresentById): self.data = data -class ExtendedMetadataIndex(RepresentById): +class ExtendedMetadataIndex(Base, RepresentById): + __tablename__ = 'extended_metadata_index' + + id = Column(Integer, primary_key=True) + extended_metadata_id = Column(Integer, + ForeignKey('extended_metadata.id', onupdate='CASCADE', ondelete='CASCADE'), index=True) + path = Column(String(255)) + value = Column(TEXT) + extended_metadata = relationship('ExtendedMetadata', back_populates='children') + def __init__(self, extended_metadata, path, value): self.extended_metadata = extended_metadata self.path = path diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 18314a19f5d..786383e4817 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -195,14 +195,6 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), Column("message", TrimmedString(255))) -model.ExtendedMetadataIndex.table = Table( - "extended_metadata_index", metadata, - Column("id", Integer, primary_key=True), - Column("extended_metadata_id", Integer, - ForeignKey("extended_metadata.id", onupdate="CASCADE", ondelete="CASCADE"), index=True), - Column("path", String(255)), - Column("value", TEXT)) - model.LibraryInfoAssociation.table = Table( "library_info_association", metadata, Column("id", Integer, primary_key=True), @@ -949,10 +941,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.ExtendedMetadataIndex, model.ExtendedMetadataIndex.table, properties=dict( - extended_metadata=relation(model.ExtendedMetadata, back_populates='children') -)) - mapper_registry.map_imperatively(model.LibraryInfoAssociation, model.LibraryInfoAssociation.table, properties=dict( library=relation(model.Library, primaryjoin=( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 4b3706d24ba..2aac1ebb843 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -514,6 +514,30 @@ class TestExtendedMetadata(BaseTest): assert stored_obj.children == [extended_metadata_index] +class TestExtendedMetadataIndex(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'extended_metadata_index' + + def test_columns(self, session, cls_, extended_metadata): + path, value = 'a', 'b' + obj = cls_(extended_metadata, path, value) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.extended_metadata_id == extended_metadata.id + assert stored_obj.path == path + assert stored_obj.value == value + + def test_relationships(self, session, cls_, extended_metadata): + obj = cls_(extended_metadata, None, None) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.extended_metadata.id == extended_metadata.id + + class TestFormDefinition(BaseTest): def test_table(self, cls_): From eac35c7a220bf5dff994f570fcaefa1cd023231e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 18:08:09 -0400 Subject: [PATCH 124/225] Map JobExportHistoryArchive declaratively --- lib/galaxy/model/__init__.py | 14 +++++++++++++- lib/galaxy/model/mapping.py | 16 +--------------- test/unit/model/test_model.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b166ef28e50..3a3d58c8d22 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1873,7 +1873,19 @@ class FakeDatasetAssociation: return isinstance(other, FakeDatasetAssociation) and self.dataset == other.dataset -class JobExportHistoryArchive(RepresentById): +class JobExportHistoryArchive(Base, RepresentById): + __tablename__ = 'job_export_history_archive' + + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + compressed = Column(Boolean, index=True, default=False) + history_attrs_filename = Column(TEXT) + job = relationship('Job') + dataset = relationship('Dataset', back_populates='job_export_history_archive') + history = relationship('History', back_populates='exports') + ATTRS_FILENAME_HISTORY = 'history_attrs.txt' def __init__(self, job=None, history=None, dataset=None, compressed=False, diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 786383e4817..e7f393f516c 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -297,15 +297,6 @@ model.JobExternalOutputMetadata.table = Table( Column("filename_override_metadata", String(255)), Column("job_runner_external_pid", String(255))) -model.JobExportHistoryArchive.table = Table( - "job_export_history_archive", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), - Column("compressed", Boolean, index=True, default=False), - Column("history_attrs_filename", TEXT)) - model.JobImportHistoryArchive.table = Table( "job_import_history_archive", metadata, Column("id", Integer, primary_key=True), @@ -840,6 +831,7 @@ simple_mapping(model.Dataset, viewonly=True), hashes=relation(model.DatasetHash, back_populates='dataset'), sources=relation(model.DatasetSource, back_populates='dataset'), + job_export_history_archive=relation(model.JobExportHistoryArchive, back_populates='dataset'), ) mapper_registry.map_imperatively(model.HistoryDatasetAssociationDisplayAtAuthorization, model.HistoryDatasetAssociationDisplayAtAuthorization.table, properties=dict( @@ -1057,12 +1049,6 @@ mapper_registry.map_imperatively(model.JobExternalOutputMetadata, model.JobExter library_dataset_dataset_association=relation(model.LibraryDatasetDatasetAssociation, lazy=False) )) -mapper_registry.map_imperatively(model.JobExportHistoryArchive, model.JobExportHistoryArchive.table, properties=dict( - job=relation(model.Job), - dataset=relation(model.Dataset, backref='job_export_history_archive'), - history=relation(model.History, back_populates='exports'), -)) - mapper_registry.map_imperatively(model.JobImportHistoryArchive, model.JobImportHistoryArchive.table, properties=dict( job=relation(model.Job), history=relation(model.History) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 2aac1ebb843..56721948d56 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1232,6 +1232,34 @@ class TestHistoryTagAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestJobExportHistoryArchive(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_export_history_archive' + + def test_columns(self, session, cls_, job, history, dataset): + compressed, history_attrs_filename = True, 'a' + obj = cls_(job, history, dataset, compressed, history_attrs_filename) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.history_id == history.id + assert stored_obj.dataset_id == dataset.id + assert stored_obj.compressed == compressed + assert stored_obj.history_attrs_filename == history_attrs_filename + + def test_relationships(self, session, cls_, job, history, dataset): + obj = cls_(job, history, dataset, True, None) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.history.id == history.id + assert stored_obj.dataset.id == dataset.id + + class TestJobMetricNumeric(BaseTest): def test_table(self, cls_): From 511b2c21334a6134d06731c6c14c9ae2a814edf2 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 18:13:23 -0400 Subject: [PATCH 125/225] Map JobImportHistoryArchive declaratively --- lib/galaxy/model/__init__.py | 11 ++++++++++- lib/galaxy/model/mapping.py | 12 ------------ test/unit/model/test_model.py | 25 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3a3d58c8d22..ad7f7a83fff 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1965,7 +1965,16 @@ class JobExportHistoryArchive(Base, RepresentById): } -class JobImportHistoryArchive(RepresentById): +class JobImportHistoryArchive(Base, RepresentById): + __tablename__ = 'job_import_history_archive' + + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + archive_dir = Column(TEXT) + job = relationship('Job') + history = relationship('History') + def __init__(self, job=None, history=None, archive_dir=None): self.job = job self.history = history diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index e7f393f516c..688796cf182 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -297,13 +297,6 @@ model.JobExternalOutputMetadata.table = Table( Column("filename_override_metadata", String(255)), Column("job_runner_external_pid", String(255))) -model.JobImportHistoryArchive.table = Table( - "job_import_history_archive", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("archive_dir", TEXT)) - model.GenomeIndexToolData.table = Table( "genome_index_tool_data", metadata, Column("id", Integer, primary_key=True), @@ -1049,11 +1042,6 @@ mapper_registry.map_imperatively(model.JobExternalOutputMetadata, model.JobExter library_dataset_dataset_association=relation(model.LibraryDatasetDatasetAssociation, lazy=False) )) -mapper_registry.map_imperatively(model.JobImportHistoryArchive, model.JobImportHistoryArchive.table, properties=dict( - job=relation(model.Job), - history=relation(model.History) -)) - mapper_registry.map_imperatively(model.GenomeIndexToolData, model.GenomeIndexToolData.table, properties=dict( job=relation(model.Job, backref='job'), dataset=relation(model.Dataset, backref='genome_index_tool_data'), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 56721948d56..5a8229ae6b1 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1260,6 +1260,31 @@ class TestJobExportHistoryArchive(BaseTest): assert stored_obj.dataset.id == dataset.id +class TestJobImportHistoryArchive(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_import_history_archive' + + def test_columns(self, session, cls_, job, history): + archive_dir = 'a' + obj = cls_(job, history, archive_dir) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.history_id == history.id + assert stored_obj. archive_dir == archive_dir + + def test_relationships(self, session, cls_, job, history, dataset): + obj = cls_(job, history) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.history.id == history.id + + class TestJobMetricNumeric(BaseTest): def test_table(self, cls_): From a0b7c72f1ea23d8760ec83f18a3ef0283fd20e49 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 18:42:47 -0400 Subject: [PATCH 126/225] Map DeferredJob declaratively --- lib/galaxy/model/__init__.py | 10 +++++++++- lib/galaxy/model/mapping.py | 11 ----------- test/unit/model/test_model.py | 23 +++++++++++++++++++++++ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ad7f7a83fff..1acc77b78b0 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2033,7 +2033,15 @@ class GenomeIndexToolData(RepresentById): self.transfer = transfer_job -class DeferredJob(RepresentById): +class DeferredJob(Base, RepresentById): + __tablename__ = 'deferred_job' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + state = Column(String(64), index=True) + plugin = Column(String(128), index=True) + params = Column(MutableJSONType) class states(str, Enum): NEW = 'new' diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 688796cf182..48fec5f188d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -352,15 +352,6 @@ model.PostJobActionAssociation.table = Table( Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=False), Column("post_job_action_id", Integer, ForeignKey("post_job_action.id"), index=True, nullable=False)) -model.DeferredJob.table = Table( - "deferred_job", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("state", String(64), index=True), - Column("plugin", String(128), index=True), - Column("params", MutableJSONType)) - model.TransferJob.table = Table( "transfer_job", metadata, Column("id", Integer, primary_key=True), @@ -1068,8 +1059,6 @@ mapper_registry.map_imperatively(model.PostJobActionAssociation, model.PostJobAc post_job_action=relation(model.PostJobAction) )) -mapper_registry.map_imperatively(model.DeferredJob, model.DeferredJob.table, properties={}) - mapper_registry.map_imperatively(model.TransferJob, model.TransferJob.table, properties={}) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 5a8229ae6b1..2a6cd69f3c6 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -410,6 +410,29 @@ class TestDefaultUserPermissions(BaseTest): assert stored_obj.role.id == role.id +class TestDeferredJob(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'deferred_job' + + def test_columns(self, session, cls_, model): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + state, plugin, params = model.DeferredJob.states.NEW, 'a', 'b' + obj = cls_(state, plugin, params) + obj.create_time = create_time + obj.update_time = update_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.state == state + assert stored_obj.plugin == plugin + assert stored_obj.params == params + + class TestDynamicTool(BaseTest): def test_table(self, cls_): From e4a8d1426caff5a38105d92fb2bc98c9f887f816 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 19:01:50 -0400 Subject: [PATCH 127/225] Map WorkflowRequestStepState declaratively --- lib/galaxy/model/__init__.py | 15 ++++++++++++--- lib/galaxy/model/mapping.py | 12 ------------ test/unit/model/test_model.py | 27 +++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1acc77b78b0..3ba4c684c7c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6595,16 +6595,25 @@ class WorkflowRequestInputParameter(Base, Dictifiable, RepresentById): self.type = type -class WorkflowRequestStepState(Dictifiable, RepresentById): +class WorkflowRequestStepState(Base, Dictifiable, RepresentById): """ Workflow step value parameters. """ + __tablename__ = 'workflow_request_step_states' + + id = Column(Integer, primary_key=True) + workflow_invocation_id = Column(Integer, + ForeignKey('workflow_invocation.id', onupdate='CASCADE', ondelete='CASCADE')) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) + value = Column(MutableJSONType) + workflow_step = relationship('WorkflowStep') + dict_collection_visible_keys = ['id', 'name', 'value', 'workflow_step_id'] def __init__(self, workflow_step=None, name=None, value=None): self.workflow_step = workflow_step - self.name = name + self.name = name # TODO this is unused and not persisted. Verify and remove. self.value = value - self.type = type + self.type = type # TODO this is incorrect. Verify and remove. class WorkflowRequestToInputDatasetAssociation(Dictifiable, RepresentById): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 48fec5f188d..7ec39cd18a6 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -482,15 +482,6 @@ model.WorkflowStepInput.table = Table( Index('ix_workflow_step_input_workflow_step_id_name_unique', "workflow_step_id", "name", unique=True, mysql_length={'name': 200}), ) - -model.WorkflowRequestStepState.table = Table( - "workflow_request_step_states", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_invocation_id", Integer, - ForeignKey("workflow_invocation.id", onupdate="CASCADE", ondelete="CASCADE")), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")), - Column("value", MutableJSONType)) - model.WorkflowRequestInputStepParameter.table = Table( "workflow_request_input_step_parameter", metadata, Column("id", Integer, primary_key=True), @@ -1288,9 +1279,6 @@ simple_mapping(model.WorkflowInvocationStep, ), ) -simple_mapping(model.WorkflowRequestStepState, - workflow_step=relation(model.WorkflowStep)) - simple_mapping(model.WorkflowRequestInputStepParameter, workflow_step=relation(model.WorkflowStep)) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 2a6cd69f3c6..654d6df1d9d 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3047,6 +3047,33 @@ class TestWorkerProcess(BaseTest): assert stored_obj.update_time == update_time +class TestWorkflowRequestStepState(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_request_step_states' + + def test_columns(self, session, cls_, workflow_step, workflow_invocation): + value = 'a' + obj = cls_(workflow_step, None, value) + obj.workflow_invocation = workflow_invocation + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.value == value + + def test_relationships(self, session, cls_, workflow_step, workflow_invocation): + obj = cls_(workflow_step) + obj.workflow_invocation = workflow_invocation + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id + assert stored_obj.workflow_step.id == workflow_step.id + + class TestWorkflowStepAnnotationAssociation(BaseTest): def test_table(self, cls_): From 768f5c67b7f86e5ce54c1a26fc39ce22b5b11961 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 19:17:48 -0400 Subject: [PATCH 128/225] Map WorkflowRequestInputStepParameter declaratively --- lib/galaxy/model/__init__.py | 10 +++++++++- lib/galaxy/model/mapping.py | 11 ----------- test/unit/model/test_model.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3ba4c684c7c..5c30993aa3d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6630,9 +6630,17 @@ class WorkflowRequestToInputDatasetCollectionAssociation(Dictifiable, RepresentB dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_collection_id', 'name'] -class WorkflowRequestInputStepParameter(Dictifiable, RepresentById): +class WorkflowRequestInputStepParameter(Base, Dictifiable, RepresentById): """ Workflow step parameter inputs. """ + __tablename__ = 'workflow_request_input_step_parameter' + + id = Column(Integer, primary_key=True) + workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id'), index=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) + parameter_value = Column(MutableJSONType) + workflow_step = relationship('WorkflowStep') + dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'parameter_value'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 7ec39cd18a6..277699b92c7 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -482,14 +482,6 @@ model.WorkflowStepInput.table = Table( Index('ix_workflow_step_input_workflow_step_id_name_unique', "workflow_step_id", "name", unique=True, mysql_length={'name': 200}), ) -model.WorkflowRequestInputStepParameter.table = Table( - "workflow_request_input_step_parameter", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")), - Column("parameter_value", MutableJSONType), -) - model.WorkflowRequestToInputDatasetAssociation.table = Table( "workflow_request_to_input_dataset", metadata, Column("id", Integer, primary_key=True), @@ -1279,9 +1271,6 @@ simple_mapping(model.WorkflowInvocationStep, ), ) -simple_mapping(model.WorkflowRequestInputStepParameter, - workflow_step=relation(model.WorkflowStep)) - simple_mapping(model.WorkflowRequestToInputDatasetAssociation, workflow_step=relation(model.WorkflowStep), dataset=relation(model.HistoryDatasetAssociation)) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 654d6df1d9d..f1e85ec798b 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3047,6 +3047,36 @@ class TestWorkerProcess(BaseTest): assert stored_obj.update_time == update_time +class TestWorkflowRequestInputStepParameter(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_request_input_step_parameter' + + def test_columns(self, session, cls_, workflow_step, workflow_invocation): + parameter_value = 'a' + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.parameter_value = parameter_value + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.parameter_value == parameter_value + + def test_relationships(self, session, cls_, workflow_step, workflow_invocation): + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id + assert stored_obj.workflow_step.id == workflow_step.id + + class TestWorkflowRequestStepState(BaseTest): def test_table(self, cls_): From 5bf646f08ddba162003eb61aa030625bec5e4700 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 19:30:42 -0400 Subject: [PATCH 129/225] Map WorkflowRequestToInputDatasetAssociation declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 13 ------------- test/unit/model/test_model.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 5c30993aa3d..9238ffc7393 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6616,9 +6616,19 @@ class WorkflowRequestStepState(Base, Dictifiable, RepresentById): self.type = type # TODO this is incorrect. Verify and remove. -class WorkflowRequestToInputDatasetAssociation(Dictifiable, RepresentById): +class WorkflowRequestToInputDatasetAssociation(Base, Dictifiable, RepresentById): """ Workflow step input dataset parameters. """ + __tablename__ = 'workflow_request_to_input_dataset' + + id = Column(Integer, primary_key=True) + name = Column(String(255)) + workflow_invocation_id = Column(Integer, ForeignKey("workflow_invocation.id"), index=True) + workflow_step_id = Column(Integer, ForeignKey("workflow_step.id")) + dataset_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + workflow_step = relationship('WorkflowStep') + dataset = relationship('HistoryDatasetAssociation') + history_content_type = "dataset" dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_id', 'name'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 277699b92c7..425c5caa7b0 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -482,14 +482,6 @@ model.WorkflowStepInput.table = Table( Index('ix_workflow_step_input_workflow_step_id_name_unique', "workflow_step_id", "name", unique=True, mysql_length={'name': 200}), ) -model.WorkflowRequestToInputDatasetAssociation.table = Table( - "workflow_request_to_input_dataset", metadata, - Column("id", Integer, primary_key=True), - Column("name", String(255)), - Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")), - Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True)) - model.WorkflowRequestToInputDatasetCollectionAssociation.table = Table( "workflow_request_to_input_collection_dataset", metadata, Column("id", Integer, primary_key=True), @@ -1271,11 +1263,6 @@ simple_mapping(model.WorkflowInvocationStep, ), ) -simple_mapping(model.WorkflowRequestToInputDatasetAssociation, - workflow_step=relation(model.WorkflowStep), - dataset=relation(model.HistoryDatasetAssociation)) - - simple_mapping(model.WorkflowRequestToInputDatasetCollectionAssociation, workflow_step=relation(model.WorkflowStep), dataset_collection=relation(model.HistoryDatasetCollectionAssociation)) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index f1e85ec798b..a3f0cd80737 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3104,6 +3104,39 @@ class TestWorkflowRequestStepState(BaseTest): assert stored_obj.workflow_step.id == workflow_step.id +class TestWorkflowRequestToInputDatasetAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_request_to_input_dataset' + + def test_columns(self, session, cls_, workflow_step, workflow_invocation, history_dataset_association): + name = 'a' + obj = cls_() + obj.name = name + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.dataset = history_dataset_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.dataset_id == history_dataset_association.id + + def test_relationships(self, session, cls_, workflow_step, workflow_invocation, history_dataset_association): + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.dataset = history_dataset_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.dataset_id == history_dataset_association.id + + class TestWorkflowStepAnnotationAssociation(BaseTest): def test_table(self, cls_): From 2b745a2428075389a9c2df2df1f3c50c6fee16ff Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 19:36:27 -0400 Subject: [PATCH 130/225] Map WorkflowRequestToInputDatasetCollectionAssoc declaratively --- lib/galaxy/model/__init__.py | 12 ++++++++- lib/galaxy/model/mapping.py | 13 ---------- test/unit/model/test_model.py | 47 +++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9238ffc7393..c21b71d5bdd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6633,9 +6633,19 @@ class WorkflowRequestToInputDatasetAssociation(Base, Dictifiable, RepresentById) dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_id', 'name'] -class WorkflowRequestToInputDatasetCollectionAssociation(Dictifiable, RepresentById): +class WorkflowRequestToInputDatasetCollectionAssociation(Base, Dictifiable, RepresentById): """ Workflow step input dataset collection parameters. """ + __tablename__ = 'workflow_request_to_input_collection_dataset' + + id = Column(Integer, primary_key=True) + name = Column(String(255)) + workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id'), index=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) + dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + workflow_step = relationship('WorkflowStep') + dataset_collection = relationship('HistoryDatasetCollectionAssociation') + history_content_type = "dataset_collection" dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_collection_id', 'name'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 425c5caa7b0..073b2913665 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -482,14 +482,6 @@ model.WorkflowStepInput.table = Table( Index('ix_workflow_step_input_workflow_step_id_name_unique', "workflow_step_id", "name", unique=True, mysql_length={'name': 200}), ) -model.WorkflowRequestToInputDatasetCollectionAssociation.table = Table( - "workflow_request_to_input_collection_dataset", metadata, - Column("id", Integer, primary_key=True), - Column("name", String(255)), - Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")), - Column("dataset_collection_id", Integer, ForeignKey("history_dataset_collection_association.id"), index=True)) - model.WorkflowStepConnection.table = Table( "workflow_step_connection", metadata, Column("id", Integer, primary_key=True), @@ -1263,11 +1255,6 @@ simple_mapping(model.WorkflowInvocationStep, ), ) -simple_mapping(model.WorkflowRequestToInputDatasetCollectionAssociation, - workflow_step=relation(model.WorkflowStep), - dataset_collection=relation(model.HistoryDatasetCollectionAssociation)) - - mapper_registry.map_imperatively(model.MetadataFile, model.MetadataFile.table, properties=dict( history_dataset=relation(model.HistoryDatasetAssociation), library_dataset=relation(model.LibraryDatasetDatasetAssociation) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index a3f0cd80737..beeed2ed3c0 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3137,6 +3137,53 @@ class TestWorkflowRequestToInputDatasetAssociation(BaseTest): assert stored_obj.dataset_id == history_dataset_association.id +class TestWorkflowRequestToInputDatasetCollectionAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_request_to_input_collection_dataset' + + def test_columns( + self, + session, + cls_, + workflow_step, + workflow_invocation, + history_dataset_collection_association + ): + name = 'a' + obj = cls_() + obj.name = name + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.dataset_collection = history_dataset_collection_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id + + def test_relationships( + self, + session, + cls_, + workflow_step, + workflow_invocation, + history_dataset_collection_association + ): + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.dataset_collection = history_dataset_collection_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id + + class TestWorkflowStepAnnotationAssociation(BaseTest): def test_table(self, cls_): From 1845f1b28b34af9f88587183d3beff0e3cbf176e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 30 Jul 2021 19:44:56 -0400 Subject: [PATCH 131/225] Map ImplicitCollectionJobs declaratively --- lib/galaxy/model/__init__.py | 6 +++++- lib/galaxy/model/mapping.py | 8 -------- test/unit/model/test_model.py | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index c21b71d5bdd..b5f3028a0e5 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1786,7 +1786,11 @@ class ImplicitlyCreatedDatasetCollectionInput(RepresentById): self.input_dataset_collection = input_dataset_collection -class ImplicitCollectionJobs(RepresentById): +class ImplicitCollectionJobs(Base, RepresentById): + __tablename__ = 'implicit_collection_jobs' + + id = Column(Integer, primary_key=True) + populated_state = Column(TrimmedString(64), default='new', nullable=False) class populated_states(str, Enum): NEW = 'new' # New implicit jobs object, unpopulated job associations diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 073b2913665..574f38c3930 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -267,12 +267,6 @@ model.ImplicitlyCreatedDatasetCollectionInput.table = Table( ForeignKey("history_dataset_collection_association.id"), index=True), Column("name", Unicode(255))) -model.ImplicitCollectionJobs.table = Table( - "implicit_collection_jobs", metadata, - Column("id", Integer, primary_key=True), - Column("populated_state", TrimmedString(64), default='new', nullable=False), -) - model.ImplicitCollectionJobsJobAssociation.table = Table( "implicit_collection_jobs_job_association", metadata, Column("id", Integer, primary_key=True), @@ -970,8 +964,6 @@ simple_mapping(model.ImplicitlyCreatedDatasetCollectionInput, ), ) -simple_mapping(model.ImplicitCollectionJobs) - # simple_mapping( # model.ImplicitCollectionJobsHistoryDatasetCollectionAssociation, # history_dataset_collection_associations=relation( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index beeed2ed3c0..c1d2b8d5660 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1255,6 +1255,22 @@ class TestHistoryTagAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestImplicitCollectionJobs(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'implicit_collection_jobs' + + def test_columns(self, session, cls_, model): + populated_state = model.ImplicitCollectionJobs.populated_states.NEW + obj = cls_() + obj.populated_state = populated_state + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.populated_state == populated_state + + class TestJobExportHistoryArchive(BaseTest): def test_table(self, cls_): From 982438540b30929f66f79f2c068cee5a1fc96f24 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 31 Jul 2021 01:10:15 -0400 Subject: [PATCH 132/225] Map HistoryDatasetAssociationDisplayAtAuthorization declaratively --- lib/galaxy/model/__init__.py | 14 +++++++++++++- lib/galaxy/model/mapping.py | 15 --------------- test/unit/model/test_model.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b5f3028a0e5..f02f230864f 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4193,7 +4193,19 @@ class HistoryDatasetAssociationHistory(Base, RepresentById): self._metadata = metadata -class HistoryDatasetAssociationDisplayAtAuthorization(RepresentById): +# hda read access permission given by a user to a specific site (gen. for external display applications) +class HistoryDatasetAssociationDisplayAtAuthorization(Base, RepresentById): + __tablename__ = 'history_dataset_association_display_at_authorization' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, index=True, default=now, onupdate=now) + history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + site = Column(TrimmedString(255)) + history_dataset_association = relationship('HistoryDatasetAssociation') + user = relationship('User') + def __init__(self, hda=None, user=None, site=None): self.history_dataset_association = hda self.user = user diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 574f38c3930..ff96e84e734 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -124,16 +124,6 @@ model.Dataset.table = Table( Column('total_size', Numeric(15, 0)), Column('uuid', UUIDType())) -# hda read access permission given by a user to a specific site (gen. for external display applications) -model.HistoryDatasetAssociationDisplayAtAuthorization.table = Table( - "history_dataset_association_display_at_authorization", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, index=True, default=now, onupdate=now), - Column("history_dataset_association_id", Integer, ForeignKey("history_dataset_association.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("site", TrimmedString(255))) - model.HistoryDatasetAssociationSubset.table = Table( "history_dataset_association_subset", metadata, Column("id", Integer, primary_key=True), @@ -779,11 +769,6 @@ simple_mapping(model.Dataset, job_export_history_archive=relation(model.JobExportHistoryArchive, back_populates='dataset'), ) -mapper_registry.map_imperatively(model.HistoryDatasetAssociationDisplayAtAuthorization, model.HistoryDatasetAssociationDisplayAtAuthorization.table, properties=dict( - history_dataset_association=relation(model.HistoryDatasetAssociation), - user=relation(model.User) -)) - mapper_registry.map_imperatively(model.HistoryDatasetAssociationSubset, model.HistoryDatasetAssociationSubset.table, properties=dict( hda=relation(model.HistoryDatasetAssociation, primaryjoin=(model.HistoryDatasetAssociationSubset.table.c.history_dataset_association_id diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index c1d2b8d5660..812f7a766f6 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1057,6 +1057,35 @@ class TestHistoryDatasetAssociationAnnotationAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestHistoryDatasetAssociationDisplayAtAuthorization(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_association_display_at_authorization' + + def test_columns(self, session, cls_, history_dataset_association, user): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + site = 'a' + obj = cls_(history_dataset_association, user, site) + obj.create_time = create_time + obj.update_time = update_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.user_id == user.id + assert stored_obj.site == site + + def test_relationships(self, session, cls_, history_dataset_association, user): + obj = cls_(history_dataset_association, user) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history_dataset_association.id == history_dataset_association.id + assert stored_obj.user.id == user.id + + class TestHistoryDatasetAssociationRatingAssociation(BaseTest): def test_table(self, cls_): From e98c98397f584c58c1f0e3764dad4afcd29355b8 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 31 Jul 2021 01:45:38 -0400 Subject: [PATCH 133/225] Map JobExternalOutputMetadata declaratively --- lib/galaxy/model/__init__.py | 18 ++++++++- lib/galaxy/model/mapping.py | 21 ---------- test/unit/model/test_model.py | 74 +++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index f02f230864f..c4f4f6a9d4b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1844,7 +1844,23 @@ class PostJobActionAssociation(RepresentById): self.post_job_action = pja -class JobExternalOutputMetadata(RepresentById): +class JobExternalOutputMetadata(Base, RepresentById): + __tablename__ = 'job_external_output_metadata' + + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True, nullable=True) + library_dataset_dataset_association_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) + is_valid = Column(Boolean, default=True) + filename_in = Column(String(255)) + filename_out = Column(String(255)) + filename_results_code = Column(String(255)) + filename_kwds = Column(String(255)) + filename_override_metadata = Column(String(255)) + job_runner_external_pid = Column(String(255)) + history_dataset_association = relationship('HistoryDatasetAssociation', lazy=False) + library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', lazy=False) + def __init__(self, job=None, dataset=None): self.job = job if isinstance(dataset, galaxy.model.HistoryDatasetAssociation): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index ff96e84e734..45215cf5e25 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -265,22 +265,6 @@ model.ImplicitCollectionJobsJobAssociation.table = Table( Column("order_index", Integer, nullable=False), ) -model.JobExternalOutputMetadata.table = Table( - "job_external_output_metadata", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("history_dataset_association_id", Integer, - ForeignKey("history_dataset_association.id"), index=True, nullable=True), - Column("library_dataset_dataset_association_id", Integer, - ForeignKey("library_dataset_dataset_association.id"), index=True, nullable=True), - Column("is_valid", Boolean, default=True), - Column("filename_in", String(255)), - Column("filename_out", String(255)), - Column("filename_results_code", String(255)), - Column("filename_kwds", String(255)), - Column("filename_override_metadata", String(255)), - Column("job_runner_external_pid", String(255))) - model.GenomeIndexToolData.table = Table( "genome_index_tool_data", metadata, Column("id", Integer, primary_key=True), @@ -972,11 +956,6 @@ simple_mapping( ), ) -mapper_registry.map_imperatively(model.JobExternalOutputMetadata, model.JobExternalOutputMetadata.table, properties=dict( - history_dataset_association=relation(model.HistoryDatasetAssociation, lazy=False), - library_dataset_dataset_association=relation(model.LibraryDatasetDatasetAssociation, lazy=False) -)) - mapper_registry.map_imperatively(model.GenomeIndexToolData, model.GenomeIndexToolData.table, properties=dict( job=relation(model.Job, backref='job'), dataset=relation(model.Dataset, backref='genome_index_tool_data'), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 812f7a766f6..950770f67fd 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1328,6 +1328,80 @@ class TestJobExportHistoryArchive(BaseTest): assert stored_obj.dataset.id == dataset.id +class TestJobExternalOutputMetadata(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_external_output_metadata' + + def test_columns( + self, + session, + cls_, + job, + history_dataset_association, + library_dataset_dataset_association + ): + is_valid = False + filename_in = 'a' + filename_out = 'b' + filename_results_code = 'c' + filename_kwds = 'd' + filename_override_metadata = 'e' + job_runner_external_pid = 'f' + + # We can only pass one dataset: an hda or a ldda, but we need to test both. + # First test passing an hda + obj = cls_(job, history_dataset_association) + + obj.is_valid = is_valid + obj.filename_in = filename_in + obj.filename_out = filename_out + obj.filename_results_code = filename_results_code + obj.filename_kwds = filename_kwds + obj.filename_override_metadata = filename_override_metadata + obj.job_runner_external_pid = job_runner_external_pid + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.is_valid == is_valid + assert stored_obj.filename_in == filename_in + assert stored_obj.filename_out == filename_out + assert stored_obj.filename_results_code == filename_results_code + assert stored_obj.filename_kwds == filename_kwds + assert stored_obj.filename_override_metadata == filename_override_metadata + assert stored_obj.job_runner_external_pid == job_runner_external_pid + + # Now pass an ldda (w/no extra fields, since we've just tested them) + obj = cls_(job, library_dataset_dataset_association) + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id + + def test_relationships( + self, + session, + cls_, + job, + history_dataset_association, + library_dataset_dataset_association + ): + # First test passing an hda + obj = cls_(job, history_dataset_association) + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.dataset.id == history_dataset_association.id + + # Now pass an ldda + obj = cls_(job, library_dataset_dataset_association) + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset.id == library_dataset_dataset_association.id + + class TestJobImportHistoryArchive(BaseTest): def test_table(self, cls_): From d3782c5304155da28ab3d8c5b11325afbc440381 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 31 Jul 2021 02:36:14 -0400 Subject: [PATCH 134/225] Map GenomeIndexToolData declaratively --- lib/galaxy/model/__init__.py | 22 +++++++++- lib/galaxy/model/mapping.py | 27 +++--------- test/unit/model/test_model.py | 77 +++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index c4f4f6a9d4b..b381f35863c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2038,7 +2038,25 @@ class InteractiveToolEntryPoint(Dictifiable, RepresentById): return False -class GenomeIndexToolData(RepresentById): +class GenomeIndexToolData(Base, RepresentById): # TODO: params arg is lost + __tablename__ = 'genome_index_tool_data' + + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + deferred_job_id = Column(Integer, ForeignKey('deferred_job.id'), index=True) + transfer_job_id = Column(Integer, ForeignKey('transfer_job.id'), index=True) + dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + fasta_path = Column(String(255)) + created_time = Column(DateTime, default=now) + modified_time = Column(DateTime, default=now, onupdate=now) + indexer = Column(String(64)) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + job = relationship('Job', back_populates='job') + dataset = relationship('Dataset', back_populates='genome_index_tool_data') + user = relationship('User') + deferred = relationship('DeferredJob', back_populates='deferred_job') + transfer = relationship('TransferJob', back_populates='transfer_job') + def __init__(self, job=None, params=None, dataset=None, deferred_job=None, transfer_job=None, fasta_path=None, created_time=None, modified_time=None, dbkey=None, user=None, indexer=None): @@ -2062,6 +2080,8 @@ class DeferredJob(Base, RepresentById): state = Column(String(64), index=True) plugin = Column(String(128), index=True) params = Column(MutableJSONType) + # TODO review attr naming: DeferredJob.deferred_job (that points to a GenomeIndexToolData object) is confusing. + deferred_job = relationship('GenomeIndexToolData', back_populates='deferred') class states(str, Enum): NEW = 'new' diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 45215cf5e25..b84bfa052e5 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -265,19 +265,6 @@ model.ImplicitCollectionJobsJobAssociation.table = Table( Column("order_index", Integer, nullable=False), ) -model.GenomeIndexToolData.table = Table( - "genome_index_tool_data", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("deferred_job_id", Integer, ForeignKey("deferred_job.id"), index=True), - Column("transfer_job_id", Integer, ForeignKey("transfer_job.id"), index=True), - Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), - Column("fasta_path", String(255)), - Column("created_time", DateTime, default=now), - Column("modified_time", DateTime, default=now, onupdate=now), - Column("indexer", String(64)), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True)) - model.InteractiveToolEntryPoint.table = Table( "interactivetool_entry_point", metadata, Column("id", Integer, primary_key=True), @@ -751,6 +738,7 @@ simple_mapping(model.Dataset, hashes=relation(model.DatasetHash, back_populates='dataset'), sources=relation(model.DatasetSource, back_populates='dataset'), job_export_history_archive=relation(model.JobExportHistoryArchive, back_populates='dataset'), + genome_index_tool_data=relation(model.GenomeIndexToolData, back_populates='dataset'), ) mapper_registry.map_imperatively(model.HistoryDatasetAssociationSubset, model.HistoryDatasetAssociationSubset.table, properties=dict( @@ -956,14 +944,6 @@ simple_mapping( ), ) -mapper_registry.map_imperatively(model.GenomeIndexToolData, model.GenomeIndexToolData.table, properties=dict( - job=relation(model.Job, backref='job'), - dataset=relation(model.Dataset, backref='genome_index_tool_data'), - user=relation(model.User), - deferred=relation(model.DeferredJob, backref='deferred_job'), - transfer=relation(model.TransferJob, backref='transfer_job') -)) - mapper_registry.map_imperatively(model.InteractiveToolEntryPoint, model.InteractiveToolEntryPoint.table, properties=dict( job=relation(model.Job, backref=backref('interactivetool_entry_points', uselist=True), uselist=False) )) @@ -982,7 +962,9 @@ mapper_registry.map_imperatively(model.PostJobActionAssociation, model.PostJobAc post_job_action=relation(model.PostJobAction) )) -mapper_registry.map_imperatively(model.TransferJob, model.TransferJob.table, properties={}) +mapper_registry.map_imperatively(model.TransferJob, model.TransferJob.table, properties=dict( + transfer_job=relation(model.GenomeIndexToolData, back_populates='transfer'), +)) simple_mapping(model.DatasetCollection, @@ -1332,6 +1314,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( state_history=relation(model.JobStateHistory, back_populates='job'), text_metrics=relation(model.JobMetricText, back_populates='job'), numeric_metrics=relation(model.JobMetricNumeric, back_populates='job'), + job=relation(model.GenomeIndexToolData, back_populates='job'), # TODO review attr naming (the functionality IS correct) )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 950770f67fd..176ff46e927 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -432,6 +432,14 @@ class TestDeferredJob(BaseTest): assert stored_obj.plugin == plugin assert stored_obj.params == params + def test_relationships(self, session, cls_, genome_index_tool_data): + obj = cls_() + obj.deferred_job.append(genome_index_tool_data) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.deferred_job == [genome_index_tool_data] + class TestDynamicTool(BaseTest): @@ -746,6 +754,57 @@ class TestGalaxySessionToHistoryAssociation(BaseTest): assert stored_obj.history.id == history.id +class TestGenomeIndexToolData(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'genome_index_tool_data' + + def test_columns(self, session, cls_, job, deferred_job, transfer_job, dataset, user): + fasta_path = 'a' + created_time = datetime.now() + modified_time = created_time + timedelta(hours=1) + indexer = 'b' + obj = cls_() + obj.job = job + obj.deferred = deferred_job + obj.transfer = transfer_job + obj.dataset = dataset + obj.fasta_path = fasta_path + obj.created_time = created_time + obj.modified_time = modified_time + obj.indexer = indexer + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.deferred_job_id == deferred_job.id + assert stored_obj.transfer_job_id == transfer_job.id + assert stored_obj.dataset_id == dataset.id + assert stored_obj.fasta_path == fasta_path + assert stored_obj.created_time == created_time + assert stored_obj.modified_time == modified_time + assert stored_obj.indexer == indexer + assert stored_obj.user_id == user.id + + def test_relationships(self, session, cls_, job, deferred_job, transfer_job, dataset, user): + obj = cls_() + obj.job = job + obj.deferred = deferred_job + obj.transfer = transfer_job + obj.dataset = dataset + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.deferred.id == deferred_job.id + assert stored_obj.transfer.id == transfer_job.id + assert stored_obj.dataset.id == dataset.id + assert stored_obj.user.id == user.id + + class TestGroupQuotaAssociation(BaseTest): def test_table(self, cls_): @@ -3501,6 +3560,12 @@ def default_quota_association(model, session, quota): yield from dbcleanup_wrapper(session, dqa) +@pytest.fixture +def deferred_job(model, session): + dj = model.DeferredJob() + yield from dbcleanup_wrapper(session, dj) + + @pytest.fixture def extended_metadata(model, session): em = model.ExtendedMetadata(None) @@ -3537,6 +3602,12 @@ def galaxy_session_history_association(model, session, galaxy_session, history): yield from dbcleanup_wrapper(session, sha) +@pytest.fixture +def genome_index_tool_data(model, session): + gitd = model.GenomeIndexToolData() + yield from dbcleanup_wrapper(session, gitd) + + @pytest.fixture def group(model, session): g = model.Group() @@ -3780,6 +3851,12 @@ def tool_tag_association(model, session): yield from dbcleanup_wrapper(session, tta) +@pytest.fixture +def transfer_job(model, session): + tj = model.TransferJob() + yield from dbcleanup_wrapper(session, tj) + + @pytest.fixture def user(model, session): u = model.User(email='test@example.com', password='password') From 6b2c4880f1301816f2739b560be89b9a9954a5ca Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 31 Jul 2021 12:55:48 -0400 Subject: [PATCH 135/225] Map TransferJob declaratively --- lib/galaxy/model/__init__.py | 15 ++++++++++++++- lib/galaxy/model/mapping.py | 17 ----------------- test/unit/model/test_model.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b381f35863c..07835b72da5 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -7451,7 +7451,20 @@ class VisualizationUserShareAssociation(UserShareAssociation): self.user = None -class TransferJob(RepresentById): +class TransferJob(Base, RepresentById): + __tablename__ = 'transfer_job' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + state = Column(String(64), index=True) + path = Column(String(1024)) + info = Column(TEXT) + pid = Column(Integer) + socket = Column(Integer) + params = Column(MutableJSONType) + transfer_job = relationship('GenomeIndexToolData', back_populates='transfer') + # These states are used both by the transfer manager's IPC and the object # state in the database. Not all states are used by both. class states(str, Enum): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index b84bfa052e5..16edd8cdb0d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -307,18 +307,6 @@ model.PostJobActionAssociation.table = Table( Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=False), Column("post_job_action_id", Integer, ForeignKey("post_job_action.id"), index=True, nullable=False)) -model.TransferJob.table = Table( - "transfer_job", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("state", String(64), index=True), - Column("path", String(1024)), - Column("info", TEXT), - Column("pid", Integer), - Column("socket", Integer), - Column("params", MutableJSONType)) - model.DatasetCollection.table = Table( "dataset_collection", metadata, Column("id", Integer, primary_key=True), @@ -962,11 +950,6 @@ mapper_registry.map_imperatively(model.PostJobActionAssociation, model.PostJobAc post_job_action=relation(model.PostJobAction) )) -mapper_registry.map_imperatively(model.TransferJob, model.TransferJob.table, properties=dict( - transfer_job=relation(model.GenomeIndexToolData, back_populates='transfer'), -)) - - simple_mapping(model.DatasetCollection, elements=relation(model.DatasetCollectionElement, primaryjoin=(model.DatasetCollection.table.c.id == model.DatasetCollectionElement.table.c.dataset_collection_id), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 176ff46e927..61c8406c73f 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -2961,6 +2961,40 @@ class TestToolTagAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestTransferJob(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'transfer_job' + + def test_columns(self, session, cls_, model): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + state, path, info, pid, socket, params = model.TransferJob.states.NEW, 'a', 'b', 2, 3, 'c' + obj = cls_(state, path, info, pid, socket, params) + obj.create_time = create_time + obj.update_time = update_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.state == state + assert stored_obj.path == path + assert stored_obj.info == info + assert stored_obj.pid == pid + assert stored_obj.socket == socket + assert stored_obj.params == params + + def test_relationships(self, session, cls_, genome_index_tool_data): + obj = cls_() + obj.transfer_job.append(genome_index_tool_data) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.transfer_job == [genome_index_tool_data] + + class TestUserAction(BaseTest): def test_table(self, cls_): From f58b33262012dbbc7350e8148a9d97a9ebc339b8 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 2 Aug 2021 15:03:37 -0400 Subject: [PATCH 136/225] Map LibraryInfoAssociation declaratively --- lib/galaxy/model/__init__.py | 24 +++++++++++++++- lib/galaxy/model/mapping.py | 22 --------------- test/unit/model/test_model.py | 52 ++++++++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 07835b72da5..f5c1d214c06 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4269,6 +4269,7 @@ class Library(Base, Dictifiable, HasName, RepresentById): synopsis = Column(TEXT) root_folder = relationship('LibraryFolder', back_populates='library_root') actions = relationship('LibraryPermissions', back_populates='library') + info_association = relationship('LibraryInfoAssociation', back_populates='library') # TODO: should this be plural? permitted_actions = get_permitted_actions(filter='LIBRARY') dict_collection_visible_keys = ['id', 'name'] @@ -4781,7 +4782,28 @@ class ExtendedMetadataIndex(Base, RepresentById): self.value = value -class LibraryInfoAssociation(RepresentById): +class LibraryInfoAssociation(Base, RepresentById): + __tablename__ = 'library_info_association' + + id = Column(Integer, primary_key=True) + library_id = Column(Integer, ForeignKey('library.id'), index=True) + form_definition_id = Column(Integer, ForeignKey('form_definition.id'), index=True) + form_values_id = Column(Integer, ForeignKey('form_values.id'), index=True) + inheritable = Column(Boolean, index=True, default=False) + deleted = Column(Boolean, index=True, default=False) + + library = relationship('Library', + primaryjoin=( + lambda: and_( + LibraryInfoAssociation.library_id == Library.id, # type: ignore + not_(LibraryInfoAssociation.deleted)) # type: ignore + ), + back_populates='info_association') + template = relationship('FormDefinition', + primaryjoin=lambda: LibraryInfoAssociation.form_definition_id == FormDefinition.id) # type: ignore + info = relationship('FormValues', + primaryjoin=lambda: LibraryInfoAssociation.form_values_id == FormValues.id) # type: ignore + def __init__(self, library, form_definition, info, inheritable=False): self.library = library self.template = form_definition diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 16edd8cdb0d..2594853cbaf 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -185,15 +185,6 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), Column("message", TrimmedString(255))) -model.LibraryInfoAssociation.table = Table( - "library_info_association", metadata, - Column("id", Integer, primary_key=True), - Column("library_id", Integer, ForeignKey("library.id"), index=True), - Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True), - Column("form_values_id", Integer, ForeignKey("form_values.id"), index=True), - Column("inheritable", Boolean, index=True, default=False), - Column("deleted", Boolean, index=True, default=False)) - model.LibraryFolderInfoAssociation.table = Table( "library_folder_info_association", metadata, Column("id", Integer, primary_key=True), @@ -823,19 +814,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.LibraryInfoAssociation, model.LibraryInfoAssociation.table, properties=dict( - library=relation(model.Library, - primaryjoin=( - (model.LibraryInfoAssociation.table.c.library_id == model.Library.id) - & (not_(model.LibraryInfoAssociation.table.c.deleted)) - ), - backref="info_association"), - template=relation(model.FormDefinition, - primaryjoin=(model.LibraryInfoAssociation.table.c.form_definition_id == model.FormDefinition.id)), - info=relation(model.FormValues, - primaryjoin=(model.LibraryInfoAssociation.table.c.form_values_id == model.FormValues.id)) -)) - mapper_registry.map_imperatively(model.LibraryFolderInfoAssociation, model.LibraryFolderInfoAssociation.table, properties=dict( folder=relation(model.LibraryFolder, primaryjoin=( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 61c8406c73f..e4c5ed4a3d3 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1893,14 +1893,24 @@ class TestLibrary(BaseTest): assert stored_obj.description == description assert stored_obj.synopsis == synopsis - def test_relationships(self, session, cls_, library_folder, library_permission): + def test_relationships( + self, + session, + cls_, + library_folder, + library_permission, + library_info_association, + ): obj = cls_(None, None, None, library_folder) obj.actions.append(library_permission) + session.add(library_info_association) # must be bound to a session for lazy load of attr. `library` (https://docs.sqlalchemy.org/en/14/errors.html#error-bhk3) + obj.info_association.append(library_info_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.root_folder.id == library_folder.id assert stored_obj.actions == [library_permission] + assert stored_obj.info_association == [library_info_association] class TestLibraryDatasetCollectionAnnotationAssociation(BaseTest): @@ -2184,6 +2194,34 @@ class TestLibraryFolderPermissions(BaseTest): assert stored_obj.role.id == role.id +class TestLibraryInfoAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_info_association' + + def test_columns(self, session, cls_, library, form_definition, form_values): + inheritable, deleted = True, True + obj = cls_(library, form_definition, form_values, inheritable) + obj.deleted = deleted + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library_id == library.id + assert stored_obj.form_definition_id == form_definition.id + assert stored_obj.form_values_id == form_values.id + assert stored_obj.inheritable == inheritable + assert stored_obj.deleted == deleted + + def test_relationships(self, session, cls_, library, form_definition, form_values): + obj = cls_(library, form_definition, form_values, None) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library.id == library.id + assert stored_obj.template.id == form_definition.id + assert stored_obj.info.id == form_values.id + + class TestLibraryPermissions(BaseTest): def test_table(self, cls_): @@ -3624,6 +3662,12 @@ def form_definition_current(model, session): yield from dbcleanup_wrapper(session, fdc) +@pytest.fixture +def form_values(model, session): + fv = model.FormValues() + yield from dbcleanup_wrapper(session, fv) + + @pytest.fixture def galaxy_session(model, session, user): s = model.GalaxySession() @@ -3786,6 +3830,12 @@ def library_folder_permission(model, session, library_folder, role): yield from dbcleanup_wrapper(session, lfp) +@pytest.fixture +def library_info_association(model, session, library, form_definition, form_values): + lia = model.LibraryInfoAssociation(library, form_definition, form_values) + yield from dbcleanup_wrapper(session, lia) + + @pytest.fixture def library_permission(model, session, library, role): lp = model.LibraryPermissions('a', library, role) From 2ca4dc31ab5d3156263c5085c93fadb5956f6269 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 3 Aug 2021 16:20:03 -0400 Subject: [PATCH 137/225] Create declarative models for Cleanup* tables For consistency: if there's a table in the db, there should be a correspondign model class. Shouldn't be any downsides. --- lib/galaxy/model/__init__.py | 89 +++++++++++++++ lib/galaxy/model/mapping.py | 59 ---------- test/unit/model/test_model.py | 203 ++++++++++++++++++++++++++++++++++ 3 files changed, 292 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index f5c1d214c06..81edf01a60a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8012,6 +8012,95 @@ def _prepare_metadata_for_serialization(id_encoder, serialization_options, metad return processed_metadata +class CleanupEvent(Base): + __tablename__ = 'cleanup_event' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + message = Column(TrimmedString(1024)) + + +class CleanupEventDatasetAssociation(Base): + __tablename__ = 'cleanup_event_dataset_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) + dataset_id = Column(Integer, ForeignKey('dataset.id'), index=True) + + +class CleanupEventMetadataFileAssociation(Base): + __tablename__ = 'cleanup_event_metadata_file_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) + metadata_file_id = Column(Integer, ForeignKey('metadata_file.id'), index=True) + + +class CleanupEventHistoryAssociation(Base): + __tablename__ = 'cleanup_event_history_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + + +class CleanupEventHistoryDatasetAssociationAssociation(Base): + __tablename__ = 'cleanup_event_hda_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) + hda_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + + +class CleanupEventLibraryAssociation(Base): + __tablename__ = 'cleanup_event_library_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) + library_id = Column(Integer, ForeignKey('library.id'), index=True) + + +class CleanupEventLibraryFolderAssociation(Base): + __tablename__ = 'cleanup_event_library_folder_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) + library_folder_id = Column(Integer, ForeignKey('library_folder.id'), index=True) + + +class CleanupEventLibraryDatasetAssociation(Base): + __tablename__ = 'cleanup_event_library_dataset_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) + library_dataset_id = Column(Integer, ForeignKey('library_dataset.id'), index=True) + + +class CleanupEventLibraryDatasetDatasetAssociationAssociation(Base): + __tablename__ = 'cleanup_event_ldda_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) + ldda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + + +class CleanupEventImplicitlyConvertedDatasetAssociationAssociation(Base): + __tablename__ = 'cleanup_event_icda_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + cleanup_event_id = Column(Integer, ForeignKey('cleanup_event.id'), index=True, nullable=True) + icda_id = Column(Integer, ForeignKey('implicitly_converted_dataset_association.id'), index=True) + + # Must be defined after PageRatingAssociation has been mapped Page.average_rating = column_property( select(func.avg(PageRatingAssociation.rating)).where(PageRatingAssociation.page_id == Page.id).scalar_subquery(), diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 2594853cbaf..9d5d499e9dd 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -587,65 +587,6 @@ model.UserPreference.table = Table( Column("name", Unicode(255), index=True), Column("value", Text)) -CleanupEvent_table = Table("cleanup_event", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("message", TrimmedString(1024))) - -CleanupEventDatasetAssociation_table = Table("cleanup_event_dataset_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("cleanup_event_id", Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True), - Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True)) - -CleanupEventMetadataFileAssociation_table = Table("cleanup_event_metadata_file_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("cleanup_event_id", Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True), - Column("metadata_file_id", Integer, ForeignKey("metadata_file.id"), index=True)) - -CleanupEventHistoryAssociation_table = Table("cleanup_event_history_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("cleanup_event_id", Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True)) - -CleanupEventHistoryDatasetAssociationAssociation_table = Table("cleanup_event_hda_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("cleanup_event_id", Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True), - Column("hda_id", Integer, ForeignKey("history_dataset_association.id"), index=True)) - -CleanupEventLibraryAssociation_table = Table("cleanup_event_library_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("cleanup_event_id", Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True), - Column("library_id", Integer, ForeignKey("library.id"), index=True)) - -CleanupEventLibraryFolderAssociation_table = Table("cleanup_event_library_folder_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("cleanup_event_id", Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True), - Column("library_folder_id", Integer, ForeignKey("library_folder.id"), index=True)) - -CleanupEventLibraryDatasetAssociation_table = Table("cleanup_event_library_dataset_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("cleanup_event_id", Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True), - Column("library_dataset_id", Integer, ForeignKey("library_dataset.id"), index=True)) - -CleanupEventLibraryDatasetDatasetAssociationAssociation_table = Table("cleanup_event_ldda_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("cleanup_event_id", Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True), - Column("ldda_id", Integer, ForeignKey("library_dataset_dataset_association.id"), index=True)) - -CleanupEventImplicitlyConvertedDatasetAssociationAssociation_table = Table("cleanup_event_icda_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("cleanup_event_id", Integer, ForeignKey("cleanup_event.id"), index=True, nullable=True), - Column("icda_id", Integer, ForeignKey("implicitly_converted_dataset_association.id"), index=True)) - # With the tables defined we can define the mappers and setup the # relationships between the model objects. diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index e4c5ed4a3d3..138bd838561 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -94,6 +94,188 @@ class TestAPIKeys(BaseTest): assert stored_obj.user.id == user.id +class TestCleanupEvent(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event' + + def test_columns(self, session, cls_): + create_time, message = datetime.now(), 'a' + obj = cls_(create_time=create_time, message=message) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.message == message + + +class TestCleanupEventDatasetAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_dataset_association' + + def test_columns(self, session, cls_, cleanup_event, dataset): + create_time = datetime.now() + obj = cls_(create_time=create_time, cleanup_event_id=cleanup_event.id, dataset_id=dataset.id) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.dataset_id == dataset.id + + +class TestCleanupEventMetadataFileAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_metadata_file_association' + + def test_columns(self, session, cls_, cleanup_event, metadata_file): + create_time = datetime.now() + obj = cls_(create_time=create_time, cleanup_event_id=cleanup_event.id, metadata_file_id=metadata_file.id) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.metadata_file_id == metadata_file.id + + +class TestCleanupEventHistoryAssociation(BaseTest): + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_history_association' + + def test_columns(self, session, cls_, cleanup_event, history): + create_time = datetime.now() + obj = cls_(create_time=create_time, cleanup_event_id=cleanup_event.id, history_id=history.id) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.history_id == history.id + + +class TestCleanupEventHistoryDatasetAssociationAssociation(BaseTest): + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_hda_association' + + def test_columns(self, session, cls_, cleanup_event, history_dataset_association): + create_time = datetime.now() + obj = cls_( + create_time=create_time, + cleanup_event_id=cleanup_event.id, + hda_id=history_dataset_association.id + ) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.hda_id == history_dataset_association.id + + +class TestCleanupEventLibraryAssociation(BaseTest): + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_library_association' + + def test_columns(self, session, cls_, cleanup_event, library): + create_time = datetime.now() + obj = cls_(create_time=create_time, cleanup_event_id=cleanup_event.id, library_id=library.id) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.library_id == library.id + + +class TestCleanupEventLibraryFolderAssociation(BaseTest): + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_library_folder_association' + + def test_columns(self, session, cls_, cleanup_event, library_folder): + create_time = datetime.now() + obj = cls_( + create_time=create_time, + cleanup_event_id=cleanup_event.id, + library_folder_id=library_folder.id + ) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.library_folder_id == library_folder.id + + +class TestCleanupEventLibraryDatasetAssociation(BaseTest): + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_library_dataset_association' + + def test_columns(self, session, cls_, cleanup_event, library_dataset): + create_time = datetime.now() + obj = cls_( + create_time=create_time, + cleanup_event_id=cleanup_event.id, + library_dataset_id=library_dataset.id + ) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.library_dataset_id == library_dataset.id + + +class TestCleanupEventLibraryDatasetDatasetAssociationAssociation(BaseTest): + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_ldda_association' + + def test_columns(self, session, cls_, cleanup_event, library_dataset_dataset_association): + create_time = datetime.now() + obj = cls_( + create_time=create_time, + cleanup_event_id=cleanup_event.id, + ldda_id=library_dataset_dataset_association.id + ) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.ldda_id == library_dataset_dataset_association.id + + +class TestCleanupEventImplicitlyConvertedDatasetAssociationAssociation(BaseTest): + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_icda_association' + + def test_columns(self, session, cls_, cleanup_event, implicitly_converted_dataset_association): + create_time = datetime.now() + obj = cls_( + create_time=create_time, + cleanup_event_id=cleanup_event.id, + icda_id=implicitly_converted_dataset_association.id + ) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.icda_id == implicitly_converted_dataset_association.id + + class TestCloudAuthz(BaseTest): def test_table(self, cls_): @@ -3569,6 +3751,12 @@ def session(model): Session.remove() # Ensures we get a new session for each test +@pytest.fixture +def cleanup_event(model, session): + ce = model.CleanupEvent() + yield from dbcleanup_wrapper(session, ce) + + @pytest.fixture def cloud_authz(model, session, user, user_authnz_token): ca = model.CloudAuthz(user.id, 'a', 'b', user_authnz_token.id, 'c') @@ -3758,6 +3946,15 @@ def history_user_share_association(model, session): yield from dbcleanup_wrapper(session, husa) +@pytest.fixture +def implicitly_converted_dataset_association(model, session, history_dataset_association): + icda = model.ImplicitlyConvertedDatasetAssociation( + dataset=history_dataset_association, + parent=history_dataset_association, # using the same dataset; should work here. + ) + yield from dbcleanup_wrapper(session, icda) + + @pytest.fixture def job(model, session): j = model.Job() @@ -3842,6 +4039,12 @@ def library_permission(model, session, library, role): yield from dbcleanup_wrapper(session, lp) +@pytest.fixture +def metadata_file(model, session): + mf = model.MetadataFile() + yield from dbcleanup_wrapper(session, mf) + + @pytest.fixture def page(model, session, user): p = model.Page() From 21f4d6ee91f99ae9eab34b0e76b78b2812031763 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 4 Aug 2021 00:18:25 -0400 Subject: [PATCH 138/225] Map InteractiveToolEntryPoint declaratively --- lib/galaxy/model/__init__.py | 21 +++++++++++- lib/galaxy/model/mapping.py | 23 +------------ test/unit/model/test_model.py | 63 +++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 81edf01a60a..fbc113fc284 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2009,7 +2009,26 @@ class JobContainerAssociation(RepresentById): self.container_info = container_info or {} -class InteractiveToolEntryPoint(Dictifiable, RepresentById): +class InteractiveToolEntryPoint(Base, Dictifiable, RepresentById): + __tablename__ = 'interactivetool_entry_point' + + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + name = Column(TEXT) + token = Column(TEXT) + tool_port = Column(Integer) + host = Column(TEXT) + port = Column(Integer) + protocol = Column(TEXT) + entry_url = Column(TEXT) + requires_domain = Column(Boolean, default=True) + info = Column(MutableJSONType, nullable=True) + configured = Column(Boolean, default=False) + deleted = Column(Boolean, default=False) + created_time = Column(DateTime, default=now) + modified_time = Column(DateTime, default=now, onupdate=now) + job = relationship('Job', back_populates='interactivetool_entry_points', uselist=False) + dict_collection_visible_keys = ['id', 'name', 'active', 'created_time', 'modified_time'] dict_element_visible_keys = ['id', 'name', 'active', 'created_time', 'modified_time'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 9d5d499e9dd..57eba0b4f41 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -256,24 +256,6 @@ model.ImplicitCollectionJobsJobAssociation.table = Table( Column("order_index", Integer, nullable=False), ) -model.InteractiveToolEntryPoint.table = Table( - "interactivetool_entry_point", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("name", TEXT), - Column("token", TEXT), - Column("tool_port", Integer), - Column("host", TEXT), - Column("port", Integer), - Column("protocol", TEXT), - Column("entry_url", TEXT), - Column("requires_domain", Boolean, default=True), - Column("info", MutableJSONType, nullable=True), - Column("configured", Boolean, default=False), - Column("deleted", Boolean, default=False), - Column("created_time", DateTime, default=now), - Column("modified_time", DateTime, default=now, onupdate=now)) - model.JobContainerAssociation.table = Table( "job_container_association", metadata, Column("id", Integer, primary_key=True), @@ -851,10 +833,6 @@ simple_mapping( ), ) -mapper_registry.map_imperatively(model.InteractiveToolEntryPoint, model.InteractiveToolEntryPoint.table, properties=dict( - job=relation(model.Job, backref=backref('interactivetool_entry_points', uselist=True), uselist=False) -)) - mapper_registry.map_imperatively(model.JobContainerAssociation, model.JobContainerAssociation.table, properties=dict( job=relation(model.Job, backref=backref('container', uselist=False), uselist=False) )) @@ -1217,6 +1195,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( text_metrics=relation(model.JobMetricText, back_populates='job'), numeric_metrics=relation(model.JobMetricNumeric, back_populates='job'), job=relation(model.GenomeIndexToolData, back_populates='job'), # TODO review attr naming (the functionality IS correct) + interactivetool_entry_points=relation(model.InteractiveToolEntryPoint, back_populates='job', uselist=True), )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 138bd838561..3c84e1afdaf 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1541,6 +1541,69 @@ class TestImplicitCollectionJobs(BaseTest): assert stored_obj.populated_state == populated_state +class TestInteractiveToolEntryPoint(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'interactivetool_entry_point' + + def test_columns(self, session, cls_, job): + name = 'a' + token = 'b' + tool_port = 1 + host = 'c' + port = 2 + protocol = 'd' + entry_url = 'e' + requires_domain = False + info = 'f' + configured = True + deleted = True + created_time = datetime.now() + modified_time = created_time + timedelta(hours=1) + + obj = cls_() + + obj.job = job + obj.name = name + obj.token = token + obj.tool_port = tool_port + obj.host = host + obj.port = port + obj.protocol = protocol + obj.entry_url = entry_url + obj.requires_domain = requires_domain + obj.info = info + obj.configured = configured + obj.deleted = deleted + obj.created_time = created_time + obj.modified_time = modified_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.name == name + assert stored_obj.token == token + assert stored_obj.tool_port == tool_port + assert stored_obj.host == host + assert stored_obj.port == port + assert stored_obj.protocol == protocol + assert stored_obj.entry_url == entry_url + assert stored_obj.requires_domain == requires_domain + assert stored_obj.info == info + assert stored_obj.configured == configured + assert stored_obj.deleted == deleted + assert stored_obj.created_time == created_time + assert stored_obj.modified_time == modified_time + + def test_relationships(self, session, cls_, job): + obj = cls_(job=job) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + + class TestJobExportHistoryArchive(BaseTest): def test_table(self, cls_): From a0135275ea46d3de065452bfa98bc6eab00155f0 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 4 Aug 2021 12:33:43 -0400 Subject: [PATCH 139/225] Map ImplicitCollectionJobsJobAssociation declaratively Do not specify uselist attr in relationship(): correct values assigned by default. --- lib/galaxy/model/__init__.py | 11 ++++++- lib/galaxy/model/mapping.py | 24 ++-------------- test/unit/model/test_model.py | 54 +++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fbc113fc284..9ac9d22876f 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1791,6 +1791,7 @@ class ImplicitCollectionJobs(Base, RepresentById): id = Column(Integer, primary_key=True) populated_state = Column(TrimmedString(64), default='new', nullable=False) + jobs = relationship('ImplicitCollectionJobsJobAssociation', back_populates='implicit_collection_jobs') class populated_states(str, Enum): NEW = 'new' # New implicit jobs object, unpopulated job associations @@ -1819,7 +1820,15 @@ class ImplicitCollectionJobs(Base, RepresentById): return rval -class ImplicitCollectionJobsJobAssociation(RepresentById): +class ImplicitCollectionJobsJobAssociation(Base, RepresentById): + __tablename__ = 'implicit_collection_jobs_job_association' + + id = Column(Integer, primary_key=True) + implicit_collection_jobs_id = Column(Integer, ForeignKey('implicit_collection_jobs.id'), index=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) # Consider making this nullable... + order_index = Column(Integer, nullable=False) + implicit_collection_jobs = relationship('ImplicitCollectionJobs', back_populates='jobs') + job = relationship('Job', back_populates='implicit_collection_jobs_association') def __init__(self): self.implicit_collection_jobs_id = None diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 57eba0b4f41..1034ac07b1c 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -248,14 +248,6 @@ model.ImplicitlyCreatedDatasetCollectionInput.table = Table( ForeignKey("history_dataset_collection_association.id"), index=True), Column("name", Unicode(255))) -model.ImplicitCollectionJobsJobAssociation.table = Table( - "implicit_collection_jobs_job_association", metadata, - Column("id", Integer, primary_key=True), - Column("implicit_collection_jobs_id", Integer, ForeignKey("implicit_collection_jobs.id"), index=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), # Consider making this nullable... - Column("order_index", Integer, nullable=False), -) - model.JobContainerAssociation.table = Table( "job_container_association", metadata, Column("id", Integer, primary_key=True), @@ -819,20 +811,6 @@ simple_mapping(model.ImplicitlyCreatedDatasetCollectionInput, # ), # ) -simple_mapping( - model.ImplicitCollectionJobsJobAssociation, - implicit_collection_jobs=relation( - model.ImplicitCollectionJobs, - backref=backref("jobs", uselist=True), - uselist=False, - ), - job=relation( - model.Job, - backref=backref("implicit_collection_jobs_association", uselist=False), - uselist=False, - ), -) - mapper_registry.map_imperatively(model.JobContainerAssociation, model.JobContainerAssociation.table, properties=dict( job=relation(model.Job, backref=backref('container', uselist=False), uselist=False) )) @@ -1196,6 +1174,8 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( numeric_metrics=relation(model.JobMetricNumeric, back_populates='job'), job=relation(model.GenomeIndexToolData, back_populates='job'), # TODO review attr naming (the functionality IS correct) interactivetool_entry_points=relation(model.InteractiveToolEntryPoint, back_populates='job', uselist=True), + implicit_collection_jobs_association=relation(model.ImplicitCollectionJobsJobAssociation, + back_populates='job', uselist=False), )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 3c84e1afdaf..1a57fc5bec3 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1540,6 +1540,47 @@ class TestImplicitCollectionJobs(BaseTest): assert stored_obj.id == obj_id assert stored_obj.populated_state == populated_state + def test_relationships(self, session, cls_, implicit_collection_jobs_job_association): + obj = cls_() + obj.jobs.append(implicit_collection_jobs_job_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.jobs == [implicit_collection_jobs_job_association] + + +class TestImplicitCollectionJobsJobAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'implicit_collection_jobs_job_association' + + def test_columns(self, session, cls_, implicit_collection_jobs, job): + order_index = 1 + obj = cls_() + obj.implicit_collection_jobs = implicit_collection_jobs + session.add(job) # TODO review this after remapping Job (required to lazy-load attr) + obj.job = job + obj.order_index = order_index + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.implicit_collection_jobs_id == implicit_collection_jobs.id + assert stored_obj.job_id == job.id + assert stored_obj.order_index == order_index + + def test_relationships(self, session, cls_, implicit_collection_jobs, job): + obj = cls_() + obj.implicit_collection_jobs = implicit_collection_jobs + session.add(job) # TODO review this after remapping Job (required to lazy-load attr) + obj.job = job + obj.order_index = 1 + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.implicit_collection_jobs.id == implicit_collection_jobs.id + assert stored_obj.job.id == job.id + class TestInteractiveToolEntryPoint(BaseTest): @@ -4009,6 +4050,19 @@ def history_user_share_association(model, session): yield from dbcleanup_wrapper(session, husa) +@pytest.fixture +def implicit_collection_jobs(model, session): + icj = model.ImplicitCollectionJobs(populated_state='new') + yield from dbcleanup_wrapper(session, icj) + + +@pytest.fixture +def implicit_collection_jobs_job_association(model, session): + icjja = model.ImplicitCollectionJobsJobAssociation() + icjja.order_index = 1 + yield from dbcleanup_wrapper(session, icjja) + + @pytest.fixture def implicitly_converted_dataset_association(model, session, history_dataset_association): icda = model.ImplicitlyConvertedDatasetAssociation( From 29353bf830effe6efdcacd1761fd12646e97dd6e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 4 Aug 2021 12:54:06 -0400 Subject: [PATCH 140/225] Map JobContainerAssociation declaratively --- lib/galaxy/model/__init__.py | 13 +++++++++++- lib/galaxy/model/mapping.py | 15 +------------- test/unit/model/test_model.py | 37 +++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9ac9d22876f..01b4494f102 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2010,7 +2010,18 @@ class JobImportHistoryArchive(Base, RepresentById): self.archive_dir = archive_dir -class JobContainerAssociation(RepresentById): +class JobContainerAssociation(Base, RepresentById): + __tablename__ = 'job_container_association' + + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True) + container_type = Column(TEXT) + container_name = Column(TEXT) + container_info = Column(MutableJSONType, nullable=True) + created_time = Column(DateTime, default=now) + modified_time = Column(DateTime, default=now, onupdate=now) + job = relationship('Job', back_populates='container') + def __init__(self, job=None, container_type=None, container_name=None, container_info=None): self.job = job self.container_type = container_type diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 1034ac07b1c..848dafda7ca 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -248,16 +248,6 @@ model.ImplicitlyCreatedDatasetCollectionInput.table = Table( ForeignKey("history_dataset_collection_association.id"), index=True), Column("name", Unicode(255))) -model.JobContainerAssociation.table = Table( - "job_container_association", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("container_type", TEXT), - Column("container_name", TEXT), - Column("container_info", MutableJSONType, nullable=True), - Column("created_time", DateTime, default=now), - Column("modified_time", DateTime, default=now, onupdate=now)) - model.PostJobAction.table = Table( "post_job_action", metadata, Column("id", Integer, primary_key=True), @@ -811,10 +801,6 @@ simple_mapping(model.ImplicitlyCreatedDatasetCollectionInput, # ), # ) -mapper_registry.map_imperatively(model.JobContainerAssociation, model.JobContainerAssociation.table, properties=dict( - job=relation(model.Job, backref=backref('container', uselist=False), uselist=False) -)) - mapper_registry.map_imperatively(model.PostJobAction, model.PostJobAction.table, properties=dict( workflow_step=relation(model.WorkflowStep, backref='post_job_actions', @@ -1176,6 +1162,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( interactivetool_entry_points=relation(model.InteractiveToolEntryPoint, back_populates='job', uselist=True), implicit_collection_jobs_association=relation(model.ImplicitCollectionJobsJobAssociation, back_populates='job', uselist=False), + container=relation('JobContainerAssociation', back_populates='job', uselist=False), )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 1a57fc5bec3..07f1031819d 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1645,6 +1645,43 @@ class TestInteractiveToolEntryPoint(BaseTest): assert stored_obj.job.id == job.id +class TestJobContainerAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_container_association' + + def test_columns(self, session, cls_, job): + container_type = 'a' + container_name = 'b' + container_info = 'c' + created_time = datetime.now() + modified_time = created_time + timedelta(hours=1) + + session.add(job) # TODO review this after remapping Job (required to lazy-load `container` attr) + + obj = cls_(job, container_type, container_name, container_info) + obj.created_time = created_time + obj.modified_time = modified_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.container_type == container_type + assert stored_obj.container_name == container_name + assert stored_obj.container_info == container_info + assert stored_obj.created_time == created_time + assert stored_obj.modified_time == modified_time + + def test_relationships(self, session, cls_, job): + session.add(job) # TODO review this after remapping Job (required to lazy-load `container` attr) + obj = cls_(job, None, None, None) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + + class TestJobExportHistoryArchive(BaseTest): def test_table(self, cls_): From 476e8132f84bf8492e39dd7c1689ad147b4553bb Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 4 Aug 2021 16:41:15 -0400 Subject: [PATCH 141/225] Map PostJobAction declaratively --- lib/galaxy/model/__init__.py | 14 +++++++++++++- lib/galaxy/model/mapping.py | 17 ++--------------- test/unit/model/test_model.py | 27 +++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 01b4494f102..0eec9f71b21 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1834,7 +1834,19 @@ class ImplicitCollectionJobsJobAssociation(Base, RepresentById): self.implicit_collection_jobs_id = None -class PostJobAction(RepresentById): +class PostJobAction(Base, RepresentById): + __tablename__ = 'post_job_action' + + id = Column(Integer, primary_key=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True, nullable=True) + action_type = Column(String(255), nullable=False) + output_name = Column(String(255), nullable=True) + action_arguments = Column(MutableJSONType, nullable=True) + workflow_step = relationship('WorkflowStep', + back_populates='post_job_actions', + primaryjoin=(lambda: WorkflowStep.id == PostJobAction.workflow_step_id) # type: ignore + ) + def __init__(self, action_type, workflow_step=None, output_name=None, action_arguments=None): self.action_type = action_type self.output_name = output_name diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 848dafda7ca..62528fbbd8f 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -248,14 +248,6 @@ model.ImplicitlyCreatedDatasetCollectionInput.table = Table( ForeignKey("history_dataset_collection_association.id"), index=True), Column("name", Unicode(255))) -model.PostJobAction.table = Table( - "post_job_action", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True, nullable=True), - Column("action_type", String(255), nullable=False), - Column("output_name", String(255), nullable=True), - Column("action_arguments", MutableJSONType, nullable=True)) - model.PostJobActionAssociation.table = Table( "post_job_action_association", metadata, Column("id", Integer, primary_key=True), @@ -801,12 +793,6 @@ simple_mapping(model.ImplicitlyCreatedDatasetCollectionInput, # ), # ) -mapper_registry.map_imperatively(model.PostJobAction, model.PostJobAction.table, properties=dict( - workflow_step=relation(model.WorkflowStep, - backref='post_job_actions', - primaryjoin=(model.WorkflowStep.table.c.id == model.PostJobAction.table.c.workflow_step_id)) -)) - mapper_registry.map_imperatively(model.PostJobActionAssociation, model.PostJobActionAssociation.table, properties=dict( post_job_action=relation(model.PostJobAction) )) @@ -913,7 +899,8 @@ mapper_registry.map_imperatively(model.WorkflowStep, model.WorkflowStep.table, p back_populates="workflow_step"), annotations=relation(model.WorkflowStepAnnotationAssociation, order_by=model.WorkflowStepAnnotationAssociation.id, - back_populates="workflow_step") + back_populates="workflow_step"), + post_job_actions=relation(model.PostJobAction, back_populates='workflow_step'), )) mapper_registry.map_imperatively(model.WorkflowStepInput, model.WorkflowStepInput.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 07f1031819d..c3511f5b48c 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -2835,6 +2835,33 @@ class TestPasswordResetToken(BaseTest): assert stored_obj.user.id == user.id +class TestPostJobAction(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'post_job_action' + + def test_columns(self, session, cls_, workflow_step): + action_type = 'a' + output_name = 'b' + action_arguments = 'c' + obj = cls_(action_type, workflow_step, output_name, action_arguments) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.action_type == action_type + assert stored_obj.output_name == output_name + assert stored_obj.action_arguments == action_arguments + + def test_relationships(self, session, cls_, workflow_step): + obj = cls_('a', workflow_step, None, None) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_step.id == workflow_step.id + + class TestPSAAssociation(BaseTest): def test_table(self, cls_): From 98bd36bec4d7a2af6bbad6a403595b865f1c1dc1 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 4 Aug 2021 17:05:00 -0400 Subject: [PATCH 142/225] Map PostJobActionAssociation declaratively --- lib/galaxy/model/__init__.py | 9 ++++++++- lib/galaxy/model/mapping.py | 10 ---------- test/unit/model/test_model.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 0eec9f71b21..6bb9539cb91 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1854,7 +1854,14 @@ class PostJobAction(Base, RepresentById): self.workflow_step = workflow_step -class PostJobActionAssociation(RepresentById): +class PostJobActionAssociation(Base, RepresentById): + __tablename__ = 'post_job_action_association' + + id = Column(Integer, primary_key=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True, nullable=False) + post_job_action_id = Column(Integer, ForeignKey('post_job_action.id'), index=True, nullable=False) + post_job_action = relationship('PostJobAction') + def __init__(self, pja, job=None, job_id=None): if job is not None: self.job = job diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 62528fbbd8f..49925b6073e 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -248,12 +248,6 @@ model.ImplicitlyCreatedDatasetCollectionInput.table = Table( ForeignKey("history_dataset_collection_association.id"), index=True), Column("name", Unicode(255))) -model.PostJobActionAssociation.table = Table( - "post_job_action_association", metadata, - Column("id", Integer, primary_key=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=False), - Column("post_job_action_id", Integer, ForeignKey("post_job_action.id"), index=True, nullable=False)) - model.DatasetCollection.table = Table( "dataset_collection", metadata, Column("id", Integer, primary_key=True), @@ -793,10 +787,6 @@ simple_mapping(model.ImplicitlyCreatedDatasetCollectionInput, # ), # ) -mapper_registry.map_imperatively(model.PostJobActionAssociation, model.PostJobActionAssociation.table, properties=dict( - post_job_action=relation(model.PostJobAction) -)) - simple_mapping(model.DatasetCollection, elements=relation(model.DatasetCollectionElement, primaryjoin=(model.DatasetCollection.table.c.id == model.DatasetCollectionElement.table.c.dataset_collection_id), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index c3511f5b48c..965cd5294a5 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -2862,6 +2862,29 @@ class TestPostJobAction(BaseTest): assert stored_obj.workflow_step.id == workflow_step.id +class TestPostJobActionAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'post_job_action_association' + + def test_columns(self, session, cls_, job, post_job_action): + obj = cls_(post_job_action, job) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.post_job_action_id == post_job_action.id + + def test_relationships(self, session, cls_, job, post_job_action): + obj = cls_(post_job_action, job) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.post_job_action.id == post_job_action.id + + class TestPSAAssociation(BaseTest): def test_table(self, cls_): @@ -4264,6 +4287,12 @@ def page_user_share_association(model, session): yield from dbcleanup_wrapper(session, pra) +@pytest.fixture +def post_job_action(model, session): + pja = model.PostJobAction('a') + yield from dbcleanup_wrapper(session, pja) + + @pytest.fixture def quota(model, session): q = model.Quota(get_random_string(), 'b') From 4bb376ba96a53a92c3ed49e63aaab688c21fd15b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 4 Aug 2021 17:29:42 -0400 Subject: [PATCH 143/225] Map ImplicitlyCreatedDatasetCollectionInput declaratively --- lib/galaxy/model/__init__.py | 14 ++++++++++++- lib/galaxy/model/mapping.py | 19 +----------------- test/unit/model/test_model.py | 38 +++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6bb9539cb91..deaf903d7f2 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1780,7 +1780,19 @@ class JobStateHistory(Base, RepresentById): self.info = job.info -class ImplicitlyCreatedDatasetCollectionInput(RepresentById): +class ImplicitlyCreatedDatasetCollectionInput(Base, RepresentById): + __tablename__ = 'implicitly_created_dataset_collection_inputs' + + id = Column(Integer, primary_key=True) + dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + input_dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + name = Column(Unicode(255)) + + input_dataset_collection = relationship('HistoryDatasetCollectionAssociation', + primaryjoin=( + lambda: HistoryDatasetCollectionAssociation.id == ImplicitlyCreatedDatasetCollectionInput.input_dataset_collection_id # type: ignore + )) + def __init__(self, name, input_dataset_collection): self.name = name self.input_dataset_collection = input_dataset_collection diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 49925b6073e..bccf043fd74 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -239,15 +239,6 @@ model.Job.table = Table( Column("params", TrimmedString(255), index=True), Column("handler", TrimmedString(255), index=True)) -model.ImplicitlyCreatedDatasetCollectionInput.table = Table( - "implicitly_created_dataset_collection_inputs", metadata, - Column("id", Integer, primary_key=True), - Column("dataset_collection_id", Integer, - ForeignKey("history_dataset_collection_association.id"), index=True), - Column("input_dataset_collection_id", Integer, - ForeignKey("history_dataset_collection_association.id"), index=True), - Column("name", Unicode(255))) - model.DatasetCollection.table = Table( "dataset_collection", metadata, Column("id", Integer, primary_key=True), @@ -770,14 +761,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) )) -simple_mapping(model.ImplicitlyCreatedDatasetCollectionInput, - input_dataset_collection=relation(model.HistoryDatasetCollectionAssociation, - primaryjoin=(model.HistoryDatasetCollectionAssociation.table.c.id - == model.ImplicitlyCreatedDatasetCollectionInput.table.c.input_dataset_collection_id), - # backref="implicitly_created_dataset_collections", - ), -) - # simple_mapping( # model.ImplicitCollectionJobsHistoryDatasetCollectionAssociation, # history_dataset_collection_associations=relation( @@ -809,7 +792,7 @@ simple_mapping(model.HistoryDatasetCollectionAssociation, uselist=False), implicit_input_collections=relation(model.ImplicitlyCreatedDatasetCollectionInput, primaryjoin=(model.HistoryDatasetCollectionAssociation.table.c.id - == model.ImplicitlyCreatedDatasetCollectionInput.table.c.dataset_collection_id), + == model.ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id), backref="dataset_collection", ), implicit_collection_jobs=relation( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 965cd5294a5..2a51effd277 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1582,6 +1582,44 @@ class TestImplicitCollectionJobsJobAssociation(BaseTest): assert stored_obj.job.id == job.id +class TestImplicitlyCreatedDatasetCollectionInput(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'implicitly_created_dataset_collection_inputs' + + def test_columns(self, session, cls_, history_dataset_collection_association, model): + name = 'a' + + hdca2 = model.HistoryDatasetCollectionAssociation() + persist(session, hdca2) + + obj = cls_(name, history_dataset_collection_association) + obj.dataset_collection_id = hdca2.id + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.input_dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.dataset_collection_id == hdca2.id + assert stored_obj.name == name + + delete_from_database(session, [hdca2]) + + def test_relationships(self, session, cls_, history_dataset_collection_association, model): + hdca2 = model.HistoryDatasetCollectionAssociation() + persist(session, hdca2) + + obj = cls_(None, history_dataset_collection_association) + obj.dataset_collection_id = hdca2.id + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.input_dataset_collection.id == history_dataset_collection_association.id + assert stored_obj.dataset_collection.id == hdca2.id + + delete_from_database(session, [hdca2]) + + class TestInteractiveToolEntryPoint(BaseTest): def test_table(self, cls_): From f472af151917494964f12e2358250f1357ddcdc7 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 4 Aug 2021 18:26:46 -0400 Subject: [PATCH 144/225] Map WorkflowStepInput declaratively --- lib/galaxy/model/__init__.py | 23 ++++++++++++++++- lib/galaxy/model/mapping.py | 26 ++----------------- test/unit/model/test_model.py | 47 +++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 25 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index deaf903d7f2..93c3c8058dd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6179,7 +6179,28 @@ class WorkflowStep(RepresentById): pass -class WorkflowStepInput(RepresentById): +class WorkflowStepInput(Base, RepresentById): + __tablename__ = 'workflow_step_input' + __table_args__ = ( + Index('ix_workflow_step_input_workflow_step_id_name_unique', + 'workflow_step_id', 'name', unique=True, mysql_length={'name': 200}), + ) + + id = Column(Integer, primary_key=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) + name = Column(TEXT) + merge_type = Column(TEXT) + scatter_type = Column(TEXT) + value_from = Column(MutableJSONType) + value_from_type = Column(TEXT) + default_value = Column(MutableJSONType) + default_value_set = Column(Boolean, default=False) + runtime_value = Column(Boolean, default=False) + workflow_step = relationship('WorkflowStep', + back_populates='inputs', + cascade='all', + primaryjoin=(lambda: WorkflowStepInput.workflow_step_id == WorkflowStep.id)) # type: ignore + default_merge_type = None default_scatter_type = None diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index bccf043fd74..55b5bf4e7fe 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -341,22 +341,6 @@ model.WorkflowStep.table = Table( # Column( "input_connections", JSONType ), Column("label", Unicode(255))) - -model.WorkflowStepInput.table = Table( - "workflow_step_input", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True), - Column("name", TEXT), - Column("merge_type", TEXT), - Column("scatter_type", TEXT), - Column("value_from", MutableJSONType), - Column("value_from_type", TEXT), - Column("default_value", MutableJSONType), - Column("default_value_set", Boolean, default=False), - Column("runtime_value", Boolean, default=False), - Index('ix_workflow_step_input_workflow_step_id_name_unique', "workflow_step_id", "name", unique=True, mysql_length={'name': 200}), -) - model.WorkflowStepConnection.table = Table( "workflow_step_connection", metadata, Column("id", Integer, primary_key=True), @@ -874,13 +858,7 @@ mapper_registry.map_imperatively(model.WorkflowStep, model.WorkflowStep.table, p order_by=model.WorkflowStepAnnotationAssociation.id, back_populates="workflow_step"), post_job_actions=relation(model.PostJobAction, back_populates='workflow_step'), -)) - -mapper_registry.map_imperatively(model.WorkflowStepInput, model.WorkflowStepInput.table, properties=dict( - workflow_step=relation(model.WorkflowStep, - backref=backref("inputs", uselist=True), - cascade="all", - primaryjoin=(model.WorkflowStepInput.table.c.workflow_step_id == model.WorkflowStep.table.c.id)) + inputs=relation(model.WorkflowStepInput, back_populates='workflow_step'), )) mapper_registry.map_imperatively(model.WorkflowOutput, model.WorkflowOutput.table, properties=dict( @@ -893,7 +871,7 @@ mapper_registry.map_imperatively(model.WorkflowStepConnection, model.WorkflowSte input_step_input=relation(model.WorkflowStepInput, backref="connections", cascade="all", - primaryjoin=(model.WorkflowStepConnection.table.c.input_step_input_id == model.WorkflowStepInput.table.c.id)), + primaryjoin=(model.WorkflowStepConnection.table.c.input_step_input_id == model.WorkflowStepInput.id)), input_subworkflow_step=relation(model.WorkflowStep, backref=backref("parent_workflow_input_connections", uselist=True), primaryjoin=(model.WorkflowStepConnection.table.c.input_subworkflow_step_id == model.WorkflowStep.table.c.id), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 2a51effd277..bab0cd313e9 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3876,6 +3876,53 @@ class TestWorkflowStepAnnotationAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestWorkflowStepInput(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_step_input' + assert has_index(cls_.__table__, ('workflow_step_id', 'name')) + + def test_columns(self, session, cls_, workflow_step): + name = 'a' + merge_type = 'b' + scatter_type = 'c' + value_from = 'd' + value_from_type = 'e' + default_value = 'f' + default_value_set = True + runtime_value = True + + obj = cls_(workflow_step) + obj.name = name + obj.merge_type = merge_type + obj.scatter_type = scatter_type + obj.value_from = value_from + obj.value_from_type = value_from_type + obj.default_value = default_value + obj.default_value_set = default_value_set + obj.runtime_value = runtime_value + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.name == name + assert stored_obj.merge_type == merge_type + assert stored_obj.scatter_type == scatter_type + assert stored_obj.value_from == value_from + assert stored_obj.value_from_type == value_from_type + assert stored_obj.default_value == default_value + assert stored_obj.default_value_set == default_value_set + assert stored_obj.runtime_value == runtime_value + + def test_relationships(self, session, cls_, workflow_step): + obj = cls_(workflow_step) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_step_id == workflow_step.id + + class TestWorkflowStepTagAssociation(BaseTest): def test_table(self, cls_): From 1cc143ecbd95443aa3ff2d0682972d4b49839d59 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 4 Aug 2021 20:35:37 -0400 Subject: [PATCH 145/225] Map WorkflowOutput declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 16 +--------------- test/unit/model/test_model.py | 25 +++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 93c3c8058dd..88832ca2131 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6261,7 +6261,17 @@ class WorkflowStepConnection(RepresentById): return copied_connection -class WorkflowOutput(RepresentById): +class WorkflowOutput(Base, RepresentById): + __tablename__ = 'workflow_output' + + id = Column(Integer, primary_key=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True, nullable=False) + output_name = Column(String(255), nullable=True) + label = Column(Unicode(255)) + uuid = Column(UUIDType) + workflow_step = relationship('WorkflowStep', + back_populates='workflow_outputs', + primaryjoin=(lambda: WorkflowStep.id == WorkflowOutput.workflow_step_id)) def __init__(self, workflow_step, output_name=None, label=None, uuid=None): self.workflow_step = workflow_step diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 55b5bf4e7fe..450b80bd6c0 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -350,15 +350,6 @@ model.WorkflowStepConnection.table = Table( Column("input_subworkflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True), ) -model.WorkflowOutput.table = Table( - "workflow_output", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True, nullable=False), - Column("output_name", String(255), nullable=True), - Column("label", Unicode(255)), - Column("uuid", UUIDType), -) - model.WorkflowInvocation.table = Table( "workflow_invocation", metadata, Column("id", Integer, primary_key=True), @@ -859,12 +850,7 @@ mapper_registry.map_imperatively(model.WorkflowStep, model.WorkflowStep.table, p back_populates="workflow_step"), post_job_actions=relation(model.PostJobAction, back_populates='workflow_step'), inputs=relation(model.WorkflowStepInput, back_populates='workflow_step'), -)) - -mapper_registry.map_imperatively(model.WorkflowOutput, model.WorkflowOutput.table, properties=dict( - workflow_step=relation(model.WorkflowStep, - backref='workflow_outputs', - primaryjoin=(model.WorkflowStep.table.c.id == model.WorkflowOutput.table.c.workflow_step_id)) + workflow_outputs=relation(model.WorkflowOutput, back_populates='workflow_step'), )) mapper_registry.map_imperatively(model.WorkflowStepConnection, model.WorkflowStepConnection.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index bab0cd313e9..c769c4ad5c2 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3708,6 +3708,31 @@ class TestWorkerProcess(BaseTest): assert stored_obj.update_time == update_time +class TestWorkflowOutput(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_output' + + def test_columns(self, session, cls_, workflow_step): + output_name, label, uuid = 'a', 'b', None + obj = cls_(workflow_step, output_name, label, uuid) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.output_name == output_name + assert stored_obj.label == label + assert stored_obj.uuid + + def test_relationships(self, session, cls_, workflow_step): + obj = cls_(workflow_step) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_step.id == workflow_step.id + + class TestWorkflowRequestInputStepParameter(BaseTest): def test_table(self, cls_): From 97d4201352bb0ae020129bf49eac5fe99d84728e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 4 Aug 2021 23:18:36 -0400 Subject: [PATCH 146/225] Map WorkflowStep declaratively --- lib/galaxy/model/__init__.py | 38 +++++++++++++++- lib/galaxy/model/mapping.py | 55 +++++------------------ test/unit/model/test_model.py | 82 +++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 47 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 88832ca2131..b8a44c47b76 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5849,7 +5849,6 @@ class Workflow(Dictifiable, RepresentById): See :class:`galaxy.model.WorkflowStep` for more information """ - dict_collection_visible_keys = ['name', 'has_cycles', 'has_errors'] dict_element_visible_keys = ['name', 'has_cycles', 'has_errors'] input_step_types = ['data_input', 'data_collection_input', 'parameter_input'] @@ -5982,12 +5981,47 @@ class Workflow(Dictifiable, RepresentById): return "Workflow[id=%d%s]" % (self.id, extra) -class WorkflowStep(RepresentById): +class WorkflowStep(Base, RepresentById): """ WorkflowStep represents a tool or subworkflow, its inputs, annotations, and any outputs that are flagged as workflow outputs. See :class:`galaxy.model.WorkflowStepInput` and :class:`galaxy.model.WorkflowStepConnection` for more information. """ + __tablename__ = 'workflow_step' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + workflow_id = Column(Integer, ForeignKey('workflow.id'), index=True, nullable=False) + subworkflow_id = Column(Integer, ForeignKey('workflow.id'), index=True, nullable=True) + dynamic_tool_id = Column(Integer, ForeignKey('dynamic_tool.id'), index=True, nullable=True) + type = Column(String(64)) + tool_id = Column(TEXT) + tool_version = Column(TEXT) + tool_inputs = Column(JSONType) + tool_errors = Column(JSONType) + position = Column(MutableJSONType) + config = Column(JSONType) + order_index = Column(Integer) + uuid = Column(UUIDType) + label = Column(Unicode(255)) + + subworkflow = relationship('Workflow', + primaryjoin=(lambda: Workflow.id == WorkflowStep.subworkflow_id), # type: ignore + back_populates='parent_workflow_steps') + dynamic_tool = relationship('DynamicTool', + primaryjoin=(lambda: DynamicTool.id == WorkflowStep.dynamic_tool_id), # type: ignore + back_populates='workflow_steps') + tags = relationship('WorkflowStepTagAssociation', + order_by=lambda: WorkflowStepTagAssociation.id, # type: ignore + back_populates='workflow_step') + annotations = relationship('WorkflowStepAnnotationAssociation', + order_by=lambda: WorkflowStepAnnotationAssociation.id, # type: ignore + back_populates="workflow_step") + post_job_actions = relationship('PostJobAction', back_populates='workflow_step') + inputs = relationship('WorkflowStepInput', back_populates='workflow_step') + workflow_outputs = relationship('WorkflowOutput', back_populates='workflow_step') + STEP_TYPE_TO_INPUT_TYPE = { "data_input": "dataset", "data_collection_input": "dataset_collection", diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 450b80bd6c0..16f2aa51ea8 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -321,26 +321,6 @@ model.Workflow.table = Table( Column("license", TEXT), Column("uuid", UUIDType, nullable=True)) -model.WorkflowStep.table = Table( - "workflow_step", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("workflow_id", Integer, ForeignKey("workflow.id"), index=True, nullable=False), - Column("subworkflow_id", Integer, ForeignKey("workflow.id"), index=True, nullable=True), - Column("dynamic_tool_id", Integer, ForeignKey("dynamic_tool.id"), index=True, nullable=True), - Column("type", String(64)), - Column("tool_id", TEXT), - Column("tool_version", TEXT), - Column("tool_inputs", JSONType), - Column("tool_errors", JSONType), - Column("position", MutableJSONType), - Column("config", JSONType), - Column("order_index", Integer), - Column("uuid", UUIDType), - # Column( "input_connections", JSONType ), - Column("label", Unicode(255))) - model.WorkflowStepConnection.table = Table( "workflow_step_connection", metadata, Column("id", Integer, primary_key=True), @@ -824,33 +804,18 @@ simple_mapping(model.DatasetCollectionElement, mapper_registry.map_imperatively(model.Workflow, model.Workflow.table, properties=dict( steps=relation(model.WorkflowStep, backref='workflow', - primaryjoin=(model.Workflow.table.c.id == model.WorkflowStep.table.c.workflow_id), - order_by=asc(model.WorkflowStep.table.c.order_index), + primaryjoin=(model.Workflow.table.c.id == model.WorkflowStep.workflow_id), + order_by=asc(model.WorkflowStep.order_index), cascade="all, delete-orphan", lazy=False), step_count=column_property( - select(func.count(model.WorkflowStep.table.c.id)).where(model.Workflow.table.c.id == model.WorkflowStep.table.c.workflow_id).scalar_subquery(), + select(func.count(model.WorkflowStep.id)).where(model.Workflow.table.c.id == model.WorkflowStep.workflow_id).scalar_subquery(), deferred=True - ) - -)) - -mapper_registry.map_imperatively(model.WorkflowStep, model.WorkflowStep.table, properties=dict( - subworkflow=relation(model.Workflow, - primaryjoin=(model.Workflow.table.c.id == model.WorkflowStep.table.c.subworkflow_id), - backref="parent_workflow_steps"), - dynamic_tool=relation(model.DynamicTool, - primaryjoin=(model.DynamicTool.id == model.WorkflowStep.table.c.dynamic_tool_id), - back_populates="workflow_steps"), - tags=relation(model.WorkflowStepTagAssociation, - order_by=model.WorkflowStepTagAssociation.id, - back_populates="workflow_step"), - annotations=relation(model.WorkflowStepAnnotationAssociation, - order_by=model.WorkflowStepAnnotationAssociation.id, - back_populates="workflow_step"), - post_job_actions=relation(model.PostJobAction, back_populates='workflow_step'), - inputs=relation(model.WorkflowStepInput, back_populates='workflow_step'), - workflow_outputs=relation(model.WorkflowOutput, back_populates='workflow_step'), + ), + parent_workflow_steps=relation( + model.WorkflowStep, + primaryjoin=(lambda: model.Workflow.id == model.WorkflowStep.subworkflow_id), # type: ignore + back_populates='subworkflow'), )) mapper_registry.map_imperatively(model.WorkflowStepConnection, model.WorkflowStepConnection.table, properties=dict( @@ -860,12 +825,12 @@ mapper_registry.map_imperatively(model.WorkflowStepConnection, model.WorkflowSte primaryjoin=(model.WorkflowStepConnection.table.c.input_step_input_id == model.WorkflowStepInput.id)), input_subworkflow_step=relation(model.WorkflowStep, backref=backref("parent_workflow_input_connections", uselist=True), - primaryjoin=(model.WorkflowStepConnection.table.c.input_subworkflow_step_id == model.WorkflowStep.table.c.id), + primaryjoin=(model.WorkflowStepConnection.table.c.input_subworkflow_step_id == model.WorkflowStep.id), ), output_step=relation(model.WorkflowStep, backref="output_connections", cascade="all", - primaryjoin=(model.WorkflowStepConnection.table.c.output_step_id == model.WorkflowStep.table.c.id)), + primaryjoin=(model.WorkflowStepConnection.table.c.output_step_id == model.WorkflowStep.id)), )) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index c769c4ad5c2..0d52e83e321 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3870,6 +3870,82 @@ class TestWorkflowRequestToInputDatasetCollectionAssociation(BaseTest): assert stored_obj.dataset_collection.id == history_dataset_collection_association.id +class TestWorkflowStep(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_step' + + def test_columns(self, session, cls_, workflow, dynamic_tool, model): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + type = 'a' + tool_id = 'b' + tool_version = 'c' + tool_inputs = 'd' + tool_errors = 'e' + position = 'f' + config = 'g' + order_index = 'h' + label = 'k' + + subworkflow = model.Workflow() + persist(session, subworkflow) + + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.workflow = workflow + obj.subworkflow = subworkflow + obj.dynamic_tool = dynamic_tool + obj.type = type + obj.tool_id = tool_id + obj.tool_version = tool_version + obj.tool_inputs = tool_inputs + obj.tool_errors = tool_errors + obj.position = position + obj.config = config + obj.order_index = order_index + obj.label = label + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.workflow_id == workflow.id + assert stored_obj.subworkflow_id == subworkflow.id + assert stored_obj.dynamic_tool_id == dynamic_tool.id + assert stored_obj.type == type + assert stored_obj.tool_id == tool_id + assert stored_obj.tool_version == tool_version + assert stored_obj.tool_inputs == tool_inputs + assert stored_obj.tool_errors == tool_errors + assert stored_obj.position == position + assert stored_obj.config == config + assert stored_obj.order_index == order_index + assert stored_obj.uuid # set in constructor + assert stored_obj.label == label + + delete_from_database(session, [subworkflow]) + + def test_relationships(self, session, cls_, workflow, dynamic_tool, model): + subworkflow = model.Workflow() + persist(session, subworkflow) + + obj = cls_() + obj.workflow = workflow + obj.subworkflow = subworkflow + obj.dynamic_tool = dynamic_tool + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow.id == workflow.id + assert stored_obj.subworkflow.id == subworkflow.id + assert stored_obj.dynamic_tool.id == dynamic_tool.id + + delete_from_database(session, [subworkflow]) + + class TestWorkflowStepAnnotationAssociation(BaseTest): def test_table(self, cls_): @@ -4127,6 +4203,12 @@ def deferred_job(model, session): yield from dbcleanup_wrapper(session, dj) +@pytest.fixture +def dynamic_tool(model, session): + dt = model.DynamicTool() + yield from dbcleanup_wrapper(session, dt) + + @pytest.fixture def extended_metadata(model, session): em = model.ExtendedMetadata(None) From 19e8fa4ac8504cd3cdff6377cf3f1d52af3e25cb Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 12:20:56 -0400 Subject: [PATCH 147/225] Map WorkflowStepConnection declaratively --- lib/galaxy/model/__init__.py | 33 ++++++++++++++- lib/galaxy/model/mapping.py | 25 ------------ test/unit/model/test_model.py | 75 +++++++++++++++++++++++++++++++++-- 3 files changed, 104 insertions(+), 29 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b8a44c47b76..30bb611a3e6 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6021,6 +6021,12 @@ class WorkflowStep(Base, RepresentById): post_job_actions = relationship('PostJobAction', back_populates='workflow_step') inputs = relationship('WorkflowStepInput', back_populates='workflow_step') workflow_outputs = relationship('WorkflowOutput', back_populates='workflow_step') + parent_workflow_input_connections = relationship('WorkflowStepConnection', + primaryjoin=(lambda: WorkflowStepConnection.input_subworkflow_step_id == WorkflowStep.id) # type: ignore + ) + output_connections = relationship('WorkflowStepConnection', + primaryjoin=(lambda: WorkflowStepConnection.output_step_id == WorkflowStep.id) # type: ignore + ) STEP_TYPE_TO_INPUT_TYPE = { "data_input": "dataset", @@ -6230,10 +6236,15 @@ class WorkflowStepInput(Base, RepresentById): default_value = Column(MutableJSONType) default_value_set = Column(Boolean, default=False) runtime_value = Column(Boolean, default=False) + workflow_step = relationship('WorkflowStep', back_populates='inputs', cascade='all', primaryjoin=(lambda: WorkflowStepInput.workflow_step_id == WorkflowStep.id)) # type: ignore + connections = relationship( + 'WorkflowStepConnection', + back_populates='input_step_input', + primaryjoin=(lambda: WorkflowStepConnection.input_step_input_id == WorkflowStepInput.id)) # type: ignore default_merge_type = None default_scatter_type = None @@ -6258,7 +6269,27 @@ class WorkflowStepInput(Base, RepresentById): return copied_step_input -class WorkflowStepConnection(RepresentById): +class WorkflowStepConnection(Base, RepresentById): + __tablename__ = 'workflow_step_connection' + + id = Column(Integer, primary_key=True) + output_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) + input_step_input_id = Column(Integer, ForeignKey('workflow_step_input.id'), index=True) + output_name = Column(TEXT) + input_subworkflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) + + input_step_input = relationship('WorkflowStepInput', + back_populates='connections', + cascade='all', + primaryjoin=(lambda: WorkflowStepConnection.input_step_input_id == WorkflowStepInput.id)) # type: ignore + input_subworkflow_step = relationship('WorkflowStep', + back_populates='parent_workflow_input_connections', + primaryjoin=(lambda: WorkflowStepConnection.input_subworkflow_step_id == WorkflowStep.id)) # type: ignore + output_step = relationship('WorkflowStep', + back_populates='output_connections', + cascade='all', + primaryjoin=(lambda: WorkflowStepConnection.output_step_id == WorkflowStep.id)) # type: ignore + # Constant used in lieu of output_name and input_name to indicate an # implicit connection between two steps that is not dependent on a dataset # or a dataset collection. Allowing for instance data manager steps to setup diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 16f2aa51ea8..89af0e6c954 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -321,15 +321,6 @@ model.Workflow.table = Table( Column("license", TEXT), Column("uuid", UUIDType, nullable=True)) -model.WorkflowStepConnection.table = Table( - "workflow_step_connection", metadata, - Column("id", Integer, primary_key=True), - Column("output_step_id", Integer, ForeignKey("workflow_step.id"), index=True), - Column("input_step_input_id", Integer, ForeignKey("workflow_step_input.id"), index=True), - Column("output_name", TEXT), - Column("input_subworkflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True), -) - model.WorkflowInvocation.table = Table( "workflow_invocation", metadata, Column("id", Integer, primary_key=True), @@ -818,22 +809,6 @@ mapper_registry.map_imperatively(model.Workflow, model.Workflow.table, propertie back_populates='subworkflow'), )) -mapper_registry.map_imperatively(model.WorkflowStepConnection, model.WorkflowStepConnection.table, properties=dict( - input_step_input=relation(model.WorkflowStepInput, - backref="connections", - cascade="all", - primaryjoin=(model.WorkflowStepConnection.table.c.input_step_input_id == model.WorkflowStepInput.id)), - input_subworkflow_step=relation(model.WorkflowStep, - backref=backref("parent_workflow_input_connections", uselist=True), - primaryjoin=(model.WorkflowStepConnection.table.c.input_subworkflow_step_id == model.WorkflowStep.id), - ), - output_step=relation(model.WorkflowStep, - backref="output_connections", - cascade="all", - primaryjoin=(model.WorkflowStepConnection.table.c.output_step_id == model.WorkflowStep.id)), -)) - - mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.table, properties=dict( user=relation(model.User, primaryjoin=(model.User.table.c.id == model.StoredWorkflow.table.c.user_id), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 0d52e83e321..bfb9ccdb6da 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3930,20 +3930,28 @@ class TestWorkflowStep(BaseTest): def test_relationships(self, session, cls_, workflow, dynamic_tool, model): subworkflow = model.Workflow() - persist(session, subworkflow) + workflow_step_connection_in = model.WorkflowStepConnection() + workflow_step_connection_out = model.WorkflowStepConnection() + extra_objects = [subworkflow, workflow_step_connection_in, workflow_step_connection_out] + for eo in extra_objects: + persist(session, eo) obj = cls_() obj.workflow = workflow obj.subworkflow = subworkflow obj.dynamic_tool = dynamic_tool + obj.parent_workflow_input_connections.append(workflow_step_connection_in) + obj.output_connections.append(workflow_step_connection_out) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.workflow.id == workflow.id assert stored_obj.subworkflow.id == subworkflow.id assert stored_obj.dynamic_tool.id == dynamic_tool.id + assert stored_obj.parent_workflow_input_connections == [workflow_step_connection_in] + assert stored_obj.output_connections == [workflow_step_connection_out] - delete_from_database(session, [subworkflow]) + delete_from_database(session, extra_objects) class TestWorkflowStepAnnotationAssociation(BaseTest): @@ -3977,6 +3985,53 @@ class TestWorkflowStepAnnotationAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestWorkflowStepConnection(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_step_connection' + + def test_columns(self, session, cls_, workflow_step_input, workflow_step, workflow, model): + output_name = 'a' + + output_workflow_step = model.WorkflowStep() + output_workflow_step.workflow = workflow + persist(session, output_workflow_step) + + obj = cls_() + obj.output_step = output_workflow_step + obj.input_step_input = workflow_step_input + obj.output_name = output_name + obj.input_subworkflow_step = workflow_step + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.output_step_id == output_workflow_step.id + assert stored_obj.input_step_input_id == workflow_step_input.id + assert stored_obj.output_name == output_name + assert stored_obj.input_subworkflow_step_id == workflow_step.id + + delete_from_database(session, [output_workflow_step]) + + def test_relationships(self, session, cls_, workflow_step_input, workflow_step, workflow, model): + output_workflow_step = model.WorkflowStep() + output_workflow_step.workflow = workflow + persist(session, output_workflow_step) + + obj = cls_() + obj.output_step = output_workflow_step + obj.input_step_input = workflow_step_input + obj.input_subworkflow_step = workflow_step + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.output_step.id == output_workflow_step.id + assert stored_obj.input_step_input.id == workflow_step_input.id + assert stored_obj.input_subworkflow_step.id == workflow_step.id + + delete_from_database(session, [output_workflow_step]) + + class TestWorkflowStepInput(BaseTest): def test_table(self, cls_): @@ -4016,12 +4071,14 @@ class TestWorkflowStepInput(BaseTest): assert stored_obj.default_value_set == default_value_set assert stored_obj.runtime_value == runtime_value - def test_relationships(self, session, cls_, workflow_step): + def test_relationships(self, session, cls_, workflow_step, workflow_step_connection): obj = cls_(workflow_step) + obj.connections.append(workflow_step_connection) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.connections == [workflow_step_connection] class TestWorkflowStepTagAssociation(BaseTest): @@ -4597,6 +4654,18 @@ def workflow_step(model, session, workflow): yield from dbcleanup_wrapper(session, s) +@pytest.fixture +def workflow_step_connection(model, session): + wsc = model.WorkflowStepConnection() + yield from dbcleanup_wrapper(session, wsc) + + +@pytest.fixture +def workflow_step_input(model, session, workflow_step): + wsi = model.WorkflowStepInput(workflow_step) + yield from dbcleanup_wrapper(session, wsi) + + @pytest.fixture def workflow_step_tag_association(model, session): wsta = model.WorkflowStepTagAssociation() From ff5b816e5fff74378e4564cc88c9473b9f5b33bf Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 15:15:38 -0400 Subject: [PATCH 148/225] Map DataManagerJobAssociation declaratively --- lib/galaxy/model/__init__.py | 14 +++++++++++++- lib/galaxy/model/mapping.py | 17 +---------------- test/unit/model/test_model.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 30bb611a3e6..3f75b6f87bb 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8112,7 +8112,19 @@ class DataManagerHistoryAssociation(RepresentById): self.user = user -class DataManagerJobAssociation(RepresentById): +class DataManagerJobAssociation(Base, RepresentById): + __tablename__ = 'data_manager_job_association' + __table_args__ = ( + Index('ix_data_manager_job_association_data_manager_id', 'data_manager_id', mysql_length=200), + ) + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, index=True, default=now, onupdate=now) + job_id = Column(Integer, ForeignKey("job.id"), index=True) + data_manager_id = Column(TEXT) + job = relationship('Job', back_populates='data_manager_association', uselist=False) + def __init__(self, id=None, job=None, data_manager_id=None): self.id = id self.job = job diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 89af0e6c954..6a31d4161b0 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -457,16 +457,6 @@ model.DataManagerHistoryAssociation.table = Table( Column("history_id", Integer, ForeignKey("history.id"), index=True), Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True)) -model.DataManagerJobAssociation.table = Table( - "data_manager_job_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, index=True, default=now, onupdate=now), - Column("job_id", Integer, ForeignKey("job.id"), index=True), - Column("data_manager_id", TEXT), - Index('ix_data_manager_job_association_data_manager_id', 'data_manager_id', mysql_length=200), -) - model.UserPreference.table = Table( "user_preference", metadata, Column("id", Integer, primary_key=True), @@ -1027,6 +1017,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( implicit_collection_jobs_association=relation(model.ImplicitCollectionJobsJobAssociation, back_populates='job', uselist=False), container=relation('JobContainerAssociation', back_populates='job', uselist=False), + data_manager_association=relation('DataManagerJobAssociation', back_populates='job', uselist=False), )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], @@ -1050,12 +1041,6 @@ mapper_registry.map_imperatively(model.DataManagerHistoryAssociation, model.Data backref='data_manager_histories') )) -mapper_registry.map_imperatively(model.DataManagerJobAssociation, model.DataManagerJobAssociation.table, properties=dict( - job=relation(model.Job, - backref=backref('data_manager_association', uselist=False), - uselist=False) -)) - class_mapper(model.HistoryDatasetCollectionAssociation).add_property( "creating_job_associations", relation(model.JobToOutputDatasetCollectionAssociation, viewonly=True)) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index bfb9ccdb6da..bdb6d82d8e4 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -307,6 +307,40 @@ class TestCloudAuthz(BaseTest): assert stored_obj.authn.id == user_authnz_token.id +class TestDataManagerJobAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'data_manager_job_association' + assert has_index(cls_.__table__, ('data_manager_id',)) + + def test_columns(self, session, cls_, job): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + data_manager_id = 'a' + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + session.add(job) + obj.job = job + obj.data_manager_id = data_manager_id + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.job_id == job.id + assert stored_obj.data_manager_id == data_manager_id + + def test_relationships(self, session, cls_, job): + session.add(job) + obj = cls_(job=job) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + + class TestDataset(BaseTest): # def test_table(self, cls_): From 91fe0498765dff3e75cd3f88d5adf42ac6346443 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 15:28:29 -0400 Subject: [PATCH 149/225] Map DataManagerHistoryAssociation declaratively --- lib/galaxy/model/__init__.py | 12 +++++++++++- lib/galaxy/model/mapping.py | 17 +---------------- test/unit/model/test_model.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3f75b6f87bb..aef3305a32f 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8105,7 +8105,17 @@ class LibraryDatasetCollectionRatingAssociation(ItemRatingAssociation, Represent # Data manager classes. -class DataManagerHistoryAssociation(RepresentById): +class DataManagerHistoryAssociation(Base, RepresentById): + __tablename__ = 'data_manager_history_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, index=True, default=now, onupdate=now) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + history = relationship('History') + user = relationship('User', back_populates='data_manager_histories') + def __init__(self, id=None, history=None, user=None): self.id = id self.history = history diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6a31d4161b0..8409621d259 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -448,15 +448,6 @@ model.VisualizationUserShareAssociation.table = Table( Column("visualization_id", Integer, ForeignKey("visualization.id"), index=True), Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True)) -# Data Manager tables -model.DataManagerHistoryAssociation.table = Table( - "data_manager_history_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, index=True, default=now, onupdate=now), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True)) - model.UserPreference.table = Table( "user_preference", metadata, Column("id", Integer, primary_key=True), @@ -612,6 +603,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( reset_tokens=relation(model.PasswordResetToken, back_populates='user'), histories_shared_by_others=relation( model.HistoryUserShareAssociation, back_populates='user'), + data_manager_histories=relation(model.DataManagerHistoryAssociation, back_populates='user'), )) # Set up proxy so that this syntax is possible: @@ -1034,13 +1026,6 @@ model.Job.any_output_dataset_collection_instances_deleted = column_property( # ) ) -# Data Manager tables -mapper_registry.map_imperatively(model.DataManagerHistoryAssociation, model.DataManagerHistoryAssociation.table, properties=dict( - history=relation(model.History), - user=relation(model.User, - backref='data_manager_histories') -)) - class_mapper(model.HistoryDatasetCollectionAssociation).add_property( "creating_job_associations", relation(model.JobToOutputDatasetCollectionAssociation, viewonly=True)) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index bdb6d82d8e4..5fdacc7f8ee 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -307,6 +307,39 @@ class TestCloudAuthz(BaseTest): assert stored_obj.authn.id == user_authnz_token.id +class TestDataManagerHistoryAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'data_manager_history_association' + + def test_columns(self, session, cls_, history, user): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.history = history + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id + + def test_relationships(self, session, cls_, history, user): + obj = cls_() + obj.history = history + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history.id == history.id + assert stored_obj.user.id == user.id + + class TestDataManagerJobAssociation(BaseTest): def test_table(self, cls_): From 65676c3f217cbfcd656a01a358e17b643eed5747 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 15:36:51 -0400 Subject: [PATCH 150/225] Map VisualizationUserShareAssoc declaratively --- lib/galaxy/model/__init__.py | 10 +++++++++- lib/galaxy/model/mapping.py | 13 ------------- test/unit/model/test_model.py | 27 +++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index aef3305a32f..2bbb6c3461b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -7633,7 +7633,15 @@ class VisualizationRevision(Base, RepresentById): ) -class VisualizationUserShareAssociation(UserShareAssociation): +class VisualizationUserShareAssociation(Base, UserShareAssociation): + __tablename__ = 'visualization_user_share_association' + + id = Column(Integer, primary_key=True) + visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user = relationship('User', backref='visualizations_shared_by_others') + visualization = relationship('Visualization', backref='users_shared_with') + def __init__(self): self.visualization = None self.user = None diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 8409621d259..9f2f19a0739 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -442,12 +442,6 @@ model.Visualization.table = Table( Index('ix_visualization_slug', 'slug', mysql_length=200), ) -model.VisualizationUserShareAssociation.table = Table( - "visualization_user_share_association", metadata, - Column("id", Integer, primary_key=True), - Column("visualization_id", Integer, ForeignKey("visualization.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True)) - model.UserPreference.table = Table( "user_preference", metadata, Column("id", Integer, primary_key=True), @@ -972,13 +966,6 @@ mapper_registry.map_imperatively(model.Visualization, model.Visualization.table, # returns a list of users that visualization is shared with. model.Visualization.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore -mapper_registry.map_imperatively(model.VisualizationUserShareAssociation, model.VisualizationUserShareAssociation.table, properties=dict( - user=relation(model.User, - backref='visualizations_shared_by_others'), - visualization=relation(model.Visualization, - backref='users_shared_with') -)) - mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( # user=relation( model.User.mapper ), user=relation(model.User), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 5fdacc7f8ee..98f779c121e 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3754,6 +3754,33 @@ class TestVisualizationRevision(BaseTest): assert stored_obj.visualization.id == visualization.id +class TestVisualizationUserShareAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'visualization_user_share_association' + + def test_columns(self, session, cls_, visualization, user): + obj = cls_() + obj.visualization = visualization + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.visualization_id == visualization.id + assert stored_obj.user_id == user.id + + def test_relationships(self, session, cls_, visualization, user): + obj = cls_() + obj.visualization = visualization + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.visualization.id == visualization.id + assert stored_obj.user.id == user.id + + class TestWorkerProcess(BaseTest): def test_table(self, cls_): From e5917dd3a240f00b9aad9f90935afbd1269ae643 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 16:08:48 -0400 Subject: [PATCH 151/225] Map WorkflowInvocationStepOutputDatasetCollectionAssoc declaratively --- lib/galaxy/model/__init__.py | 16 ++++++++++- lib/galaxy/model/mapping.py | 19 ++----------- test/unit/model/test_model.py | 53 +++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 2bbb6c3461b..a84b3c75b6d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6926,8 +6926,22 @@ class WorkflowInvocationStepOutputDatasetAssociation(Dictifiable, RepresentById) dict_collection_visible_keys = ['id', 'workflow_invocation_step_id', 'dataset_id', 'output_name'] -class WorkflowInvocationStepOutputDatasetCollectionAssociation(Dictifiable, RepresentById): +class WorkflowInvocationStepOutputDatasetCollectionAssociation(Base, Dictifiable, RepresentById): """Represents links to output dataset collections for the workflow.""" + __tablename__ = 'workflow_invocation_step_output_dataset_collection_association' + + id = Column(Integer, primary_key=True) + workflow_invocation_step_id = Column( + Integer, ForeignKey('workflow_invocation_step.id', name='fk_wisodca_wisi'), index=True) + workflow_step_id = Column( + Integer, ForeignKey('workflow_step.id', name='fk_wisodca_wsi'), index=True) + dataset_collection_id = Column( + Integer, ForeignKey('history_dataset_collection_association.id', name='fk_wisodca_dci'), index=True) + output_name = Column(String(255), nullable=True) + + workflow_invocation_step = relationship('WorkflowInvocationStep', back_populates='output_dataset_collections') + dataset_collection = relationship('HistoryDatasetCollectionAssociation') + dict_collection_visible_keys = ['id', 'workflow_invocation_step_id', 'dataset_collection_id', 'output_name'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 9f2f19a0739..5125c1d0651 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -380,15 +380,6 @@ model.WorkflowInvocationStepOutputDatasetAssociation.table = Table( Column("output_name", String(255), nullable=True), ) -model.WorkflowInvocationStepOutputDatasetCollectionAssociation.table = Table( - "workflow_invocation_step_output_dataset_collection_association", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_invocation_step_id", Integer, ForeignKey("workflow_invocation_step.id", name='fk_wisodca_wisi'), index=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id", name='fk_wisodca_wsi'), index=True), - Column("dataset_collection_id", Integer, ForeignKey("history_dataset_collection_association.id", name='fk_wisodca_dci'), index=True), - Column("output_name", String(255), nullable=True), -) - model.WorkflowInvocationToSubworkflowInvocationAssociation.table = Table( "workflow_invocation_to_subworkflow_invocation_association", metadata, Column("id", Integer, primary_key=True), @@ -875,6 +866,9 @@ simple_mapping(model.WorkflowInvocationStep, model.WorkflowInvocationToSubworkflowInvocationAssociation.table.c.workflow_step_id == model.WorkflowInvocationStep.table.c.workflow_step_id, )).scalar_subquery(), ), + output_dataset_collections=relation( + model.WorkflowInvocationStepOutputDatasetCollectionAssociation, + back_populates='workflow_invocation_step'), ) mapper_registry.map_imperatively(model.MetadataFile, model.MetadataFile.table, properties=dict( @@ -924,13 +918,6 @@ simple_mapping( dataset=relation(model.HistoryDatasetAssociation), ) - -simple_mapping( - model.WorkflowInvocationStepOutputDatasetCollectionAssociation, - workflow_invocation_step=relation(model.WorkflowInvocationStep, backref="output_dataset_collections"), - dataset_collection=relation(model.HistoryDatasetCollectionAssociation), -) - # Set up proxy so that # Page.users_shared_with # returns a list of users that page is shared with. diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 98f779c121e..06b745f9f5b 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3802,6 +3802,51 @@ class TestWorkerProcess(BaseTest): assert stored_obj.update_time == update_time +class TestWorkflowInvocationStepOutputDatasetCollectionAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_invocation_step_output_dataset_collection_association' + + def test_columns( + self, + session, + cls_, + workflow_invocation_step, + workflow_step, + history_dataset_collection_association, + ): + output_name = 'a' + obj = cls_() + obj.workflow_invocation_step = workflow_invocation_step + obj.workflow_step_id = workflow_step.id + obj.dataset_collection = history_dataset_collection_association + obj.output_name = output_name + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_step_id == workflow_invocation_step.id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.output_name == output_name + + def test_relationships( + self, + session, + cls_, + workflow_invocation_step, + history_dataset_collection_association, + ): + obj = cls_() + obj.workflow_invocation_step = workflow_invocation_step + obj.dataset_collection = history_dataset_collection_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation_step.id == workflow_invocation_step.id + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id + + class TestWorkflowOutput(BaseTest): def test_table(self, cls_): @@ -4741,6 +4786,14 @@ def workflow_invocation(model, session, workflow): yield from dbcleanup_wrapper(session, wi) +@pytest.fixture +def workflow_invocation_step(model, session, workflow_invocation, workflow_step): + wis = model.WorkflowInvocationStep() + wis.workflow_invocation = workflow_invocation + wis.workflow_step = workflow_step + yield from dbcleanup_wrapper(session, wis) + + @pytest.fixture def workflow_step(model, session, workflow): s = model.WorkflowStep() From 89fbd453a2f2678a1108a8e1e35120b08004ae28 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 16:18:02 -0400 Subject: [PATCH 152/225] Map WorkflowInvocationStepOutputDatasetAssoc declaratively --- lib/galaxy/model/__init__.py | 11 ++++++++++- lib/galaxy/model/mapping.py | 17 ++--------------- test/unit/model/test_model.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index a84b3c75b6d..664a8de58ff 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6921,8 +6921,17 @@ class WorkflowInvocationOutputValue(Dictifiable, RepresentById): dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'value'] -class WorkflowInvocationStepOutputDatasetAssociation(Dictifiable, RepresentById): +class WorkflowInvocationStepOutputDatasetAssociation(Base, Dictifiable, RepresentById): """Represents links to output datasets for the workflow.""" + __tablename__ = 'workflow_invocation_step_output_dataset_association' + + id = Column(Integer, primary_key=True) + workflow_invocation_step_id = Column(Integer, ForeignKey('workflow_invocation_step.id'), index=True) + dataset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + output_name = Column(String(255), nullable=True) + workflow_invocation_step = relationship('WorkflowInvocationStep', back_populates='output_datasets') + dataset = relationship('HistoryDatasetAssociation') + dict_collection_visible_keys = ['id', 'workflow_invocation_step_id', 'dataset_id', 'output_name'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 5125c1d0651..0c9326392c2 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -372,14 +372,6 @@ model.WorkflowInvocationOutputValue.table = Table( Column("value", MutableJSONType), ) -model.WorkflowInvocationStepOutputDatasetAssociation.table = Table( - "workflow_invocation_step_output_dataset_association", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_invocation_step_id", Integer, ForeignKey("workflow_invocation_step.id"), index=True), - Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True), - Column("output_name", String(255), nullable=True), -) - model.WorkflowInvocationToSubworkflowInvocationAssociation.table = Table( "workflow_invocation_to_subworkflow_invocation_association", metadata, Column("id", Integer, primary_key=True), @@ -869,6 +861,8 @@ simple_mapping(model.WorkflowInvocationStep, output_dataset_collections=relation( model.WorkflowInvocationStepOutputDatasetCollectionAssociation, back_populates='workflow_invocation_step'), + output_datasets=relation(model.WorkflowInvocationStepOutputDatasetAssociation, + back_populates='workflow_invocation_step'), ) mapper_registry.map_imperatively(model.MetadataFile, model.MetadataFile.table, properties=dict( @@ -911,13 +905,6 @@ simple_mapping( workflow_output=relation(model.WorkflowOutput), ) - -simple_mapping( - model.WorkflowInvocationStepOutputDatasetAssociation, - workflow_invocation_step=relation(model.WorkflowInvocationStep, backref="output_datasets"), - dataset=relation(model.HistoryDatasetAssociation), -) - # Set up proxy so that # Page.users_shared_with # returns a list of users that page is shared with. diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 06b745f9f5b..468581c567d 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3802,6 +3802,36 @@ class TestWorkerProcess(BaseTest): assert stored_obj.update_time == update_time +class TestWorkflowInvocationStepOutputDatasetAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_invocation_step_output_dataset_association' + + def test_columns(self, session, cls_, workflow_invocation_step, history_dataset_association): + output_name = 'a' + obj = cls_() + obj.workflow_invocation_step = workflow_invocation_step + obj.dataset = history_dataset_association + obj.output_name = output_name + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_step_id == workflow_invocation_step.id + assert stored_obj.dataset_id == history_dataset_association.id + assert stored_obj.output_name == output_name + + def test_relationships(self, session, cls_, workflow_invocation_step, history_dataset_association): + obj = cls_() + obj.workflow_invocation_step = workflow_invocation_step + obj.dataset = history_dataset_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation_step.id == workflow_invocation_step.id + assert stored_obj.dataset.id == history_dataset_association.id + + class TestWorkflowInvocationStepOutputDatasetCollectionAssociation(BaseTest): def test_table(self, cls_): From 619dddc132d66971978c40a86f70ddf782ee1527 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 17:19:11 -0400 Subject: [PATCH 153/225] Map WorkflowInvocationOutputDatasetCollectionAssoc declaratively --- lib/galaxy/model/__init__.py | 15 ++++++++- lib/galaxy/model/mapping.py | 22 ++------------ test/unit/model/test_model.py | 57 +++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 664a8de58ff..721e841ae3d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6910,8 +6910,21 @@ class WorkflowInvocationOutputDatasetAssociation(Dictifiable, RepresentById): dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_id', 'name'] -class WorkflowInvocationOutputDatasetCollectionAssociation(Dictifiable, RepresentById): +class WorkflowInvocationOutputDatasetCollectionAssociation(Base, Dictifiable, RepresentById): """Represents links to output dataset collections for the workflow.""" + __tablename__ = 'workflow_invocation_output_dataset_collection_association' + + id = Column(Integer, primary_key=True) + workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id', name='fk_wiodca_wii'), index=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id', name='fk_wiodca_wsi'), index=True) + dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id', name='fk_wiodca_dci'), index=True) + workflow_output_id = Column(Integer, ForeignKey('workflow_output.id', name='fk_wiodca_woi'), index=True) + + workflow_invocation = relationship('WorkflowInvocation', back_populates='output_dataset_collections') + workflow_step = relationship('WorkflowStep') + dataset_collection = relationship('HistoryDatasetCollectionAssociation') + workflow_output = relationship('WorkflowOutput') + history_content_type = "dataset_collection" dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_collection_id', 'name'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 0c9326392c2..16f16148337 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -354,15 +354,6 @@ model.WorkflowInvocationOutputDatasetAssociation.table = Table( Column("workflow_output_id", Integer, ForeignKey("workflow_output.id"), index=True), ) -model.WorkflowInvocationOutputDatasetCollectionAssociation.table = Table( - "workflow_invocation_output_dataset_collection_association", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id", name='fk_wiodca_wii'), index=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id", name='fk_wiodca_wsi'), index=True), - Column("dataset_collection_id", Integer, ForeignKey("history_dataset_collection_association.id", name='fk_wiodca_dci'), index=True), - Column("workflow_output_id", Integer, ForeignKey("workflow_output.id", name='fk_wiodca_woi'), index=True), -) - model.WorkflowInvocationOutputValue.table = Table( "workflow_invocation_output_value", metadata, Column("id", Integer, primary_key=True), @@ -836,7 +827,9 @@ mapper_registry.map_imperatively(model.WorkflowInvocation, model.WorkflowInvocat ), steps=relation(model.WorkflowInvocationStep, backref="workflow_invocation"), - workflow=relation(model.Workflow) + workflow=relation(model.Workflow), + output_dataset_collections=relation(model.WorkflowInvocationOutputDatasetCollectionAssociation, + back_populates='workflow_invocation'), )) mapper_registry.map_imperatively(model.WorkflowInvocationToSubworkflowInvocationAssociation, model.WorkflowInvocationToSubworkflowInvocationAssociation.table, properties=dict( @@ -880,15 +873,6 @@ simple_mapping( ) -simple_mapping( - model.WorkflowInvocationOutputDatasetCollectionAssociation, - workflow_invocation=relation(model.WorkflowInvocation, backref="output_dataset_collections"), - workflow_step=relation(model.WorkflowStep), - dataset_collection=relation(model.HistoryDatasetCollectionAssociation), - workflow_output=relation(model.WorkflowOutput), -) - - simple_mapping( model.WorkflowInvocationOutputValue, workflow_invocation=relation(model.WorkflowInvocation, backref="output_values"), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 468581c567d..33b88843bea 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3802,6 +3802,57 @@ class TestWorkerProcess(BaseTest): assert stored_obj.update_time == update_time +class TestWorkflowInvocationOutputDatasetCollectionAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_invocation_output_dataset_collection_association' + + def test_columns( + self, + session, + cls_, + workflow_invocation, + workflow_step, + history_dataset_collection_association, + workflow_output, + ): + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.dataset_collection = history_dataset_collection_association + obj.workflow_output = workflow_output + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.dataset_collection_id == history_dataset_collection_association.id + assert stored_obj.workflow_output_id == workflow_output.id + + def test_relationships( + self, + session, + cls_, + workflow_invocation, + workflow_step, + history_dataset_collection_association, + workflow_output, + ): + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.dataset_collection = history_dataset_collection_association + obj.workflow_output = workflow_output + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.dataset_collection.id == history_dataset_collection_association.id + assert stored_obj.workflow_output.id == workflow_output.id + + class TestWorkflowInvocationStepOutputDatasetAssociation(BaseTest): def test_table(self, cls_): @@ -4824,6 +4875,12 @@ def workflow_invocation_step(model, session, workflow_invocation, workflow_step) yield from dbcleanup_wrapper(session, wis) +@pytest.fixture +def workflow_output(model, session, workflow_step): + wo = model.WorkflowOutput(workflow_step) + yield from dbcleanup_wrapper(session, wo) + + @pytest.fixture def workflow_step(model, session, workflow): s = model.WorkflowStep() From 830cb3ece5eb1b69b5484edee9bf471539d3bcde Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 17:27:03 -0400 Subject: [PATCH 154/225] Map WorkflowInvocationOutputDatasetAssoc declaratively --- lib/galaxy/model/__init__.py | 15 ++++++++++- lib/galaxy/model/mapping.py | 21 ++------------- test/unit/model/test_model.py | 51 +++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 721e841ae3d..8d1af94550d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6904,8 +6904,21 @@ class WorkflowRequestInputStepParameter(Base, Dictifiable, RepresentById): dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'parameter_value'] -class WorkflowInvocationOutputDatasetAssociation(Dictifiable, RepresentById): +class WorkflowInvocationOutputDatasetAssociation(Base, Dictifiable, RepresentById): """Represents links to output datasets for the workflow.""" + __tablename__ = 'workflow_invocation_output_dataset_association' + + id = Column(Integer, primary_key=True) + workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id'), index=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id'), index=True) + dataset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + workflow_output_id = Column(Integer, ForeignKey('workflow_output.id'), index=True) + + workflow_invocation = relationship('WorkflowInvocation', back_populates='output_datasets') + workflow_step = relationship('WorkflowStep') + dataset = relationship('HistoryDatasetAssociation') + workflow_output = relationship('WorkflowOutput') + history_content_type = "dataset" dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_id', 'name'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 16f16148337..2bbc9703ae2 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -345,15 +345,6 @@ model.WorkflowInvocationStep.table = Table( Column("implicit_collection_jobs_id", Integer, ForeignKey("implicit_collection_jobs.id"), index=True, nullable=True), Column("action", MutableJSONType, nullable=True)) -model.WorkflowInvocationOutputDatasetAssociation.table = Table( - "workflow_invocation_output_dataset_association", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True), - Column("dataset_id", Integer, ForeignKey("history_dataset_association.id"), index=True), - Column("workflow_output_id", Integer, ForeignKey("workflow_output.id"), index=True), -) - model.WorkflowInvocationOutputValue.table = Table( "workflow_invocation_output_value", metadata, Column("id", Integer, primary_key=True), @@ -830,6 +821,8 @@ mapper_registry.map_imperatively(model.WorkflowInvocation, model.WorkflowInvocat workflow=relation(model.Workflow), output_dataset_collections=relation(model.WorkflowInvocationOutputDatasetCollectionAssociation, back_populates='workflow_invocation'), + output_datasets=relation(model.WorkflowInvocationOutputDatasetAssociation, + back_populates='workflow_invocation'), )) mapper_registry.map_imperatively(model.WorkflowInvocationToSubworkflowInvocationAssociation, model.WorkflowInvocationToSubworkflowInvocationAssociation.table, properties=dict( @@ -863,16 +856,6 @@ mapper_registry.map_imperatively(model.MetadataFile, model.MetadataFile.table, p library_dataset=relation(model.LibraryDatasetDatasetAssociation) )) - -simple_mapping( - model.WorkflowInvocationOutputDatasetAssociation, - workflow_invocation=relation(model.WorkflowInvocation, backref="output_datasets"), - workflow_step=relation(model.WorkflowStep), - dataset=relation(model.HistoryDatasetAssociation), - workflow_output=relation(model.WorkflowOutput), -) - - simple_mapping( model.WorkflowInvocationOutputValue, workflow_invocation=relation(model.WorkflowInvocation, backref="output_values"), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 33b88843bea..7e442498e75 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3802,6 +3802,57 @@ class TestWorkerProcess(BaseTest): assert stored_obj.update_time == update_time +class TestWorkflowInvocationOutputDatasetAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_invocation_output_dataset_association' + + def test_columns( + self, + session, + cls_, + workflow_invocation, + workflow_step, + history_dataset_association, + workflow_output, + ): + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.dataset = history_dataset_association + obj.workflow_output = workflow_output + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.dataset_id == history_dataset_association.id + assert stored_obj.workflow_output_id == workflow_output.id + + def test_relationships( + self, + session, + cls_, + workflow_invocation, + workflow_step, + history_dataset_association, + workflow_output, + ): + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.dataset = history_dataset_association + obj.workflow_output = workflow_output + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.dataset.id == history_dataset_association.id + assert stored_obj.workflow_output.id == workflow_output.id + + class TestWorkflowInvocationOutputDatasetCollectionAssociation(BaseTest): def test_table(self, cls_): From 9cbbdf7dd586ea935f57f30382038a32014a3e11 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 17:50:36 -0400 Subject: [PATCH 155/225] Map WorkflowInvocationToSubworkflowInvocationAssoc declaratively --- lib/galaxy/model/__init__.py | 17 ++++++++++++++++- lib/galaxy/model/mapping.py | 29 ++++++++--------------------- test/unit/model/test_model.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 8d1af94550d..0471eb6d9bd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6723,7 +6723,22 @@ class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable, RepresentById): return f"{self.__class__.__name__}[{extra}]" -class WorkflowInvocationToSubworkflowInvocationAssociation(Dictifiable, RepresentById): +class WorkflowInvocationToSubworkflowInvocationAssociation(Base, Dictifiable, RepresentById): + __tablename__ = 'workflow_invocation_to_subworkflow_invocation_association' + + id = Column(Integer, primary_key=True) + workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id', name='fk_wfi_swi_wfi'), index=True) + subworkflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id', name='fk_wfi_swi_swi'), index=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id', name='fk_wfi_swi_ws')) + + subworkflow_invocation = relationship('WorkflowInvocation', + primaryjoin=(lambda: + WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id == WorkflowInvocation.id), # type: ignore + back_populates='parent_workflow_invocation_association', + uselist=False, + ) + workflow_step = relationship('WorkflowStep') + dict_collection_visible_keys = ['id', 'workflow_step_id', 'workflow_invocation_id', 'subworkflow_invocation_id'] dict_element_visible_keys = ['id', 'workflow_step_id', 'workflow_invocation_id', 'subworkflow_invocation_id'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 2bbc9703ae2..70014ebfbfb 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -354,14 +354,6 @@ model.WorkflowInvocationOutputValue.table = Table( Column("value", MutableJSONType), ) -model.WorkflowInvocationToSubworkflowInvocationAssociation.table = Table( - "workflow_invocation_to_subworkflow_invocation_association", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id", name='fk_wfi_swi_wfi'), index=True), - Column("subworkflow_invocation_id", Integer, ForeignKey("workflow_invocation.id", name='fk_wfi_swi_swi'), index=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id", name='fk_wfi_swi_ws')), -) - model.StoredWorkflowUserShareAssociation.table = Table( "stored_workflow_user_share_connection", metadata, Column("id", Integer, primary_key=True), @@ -812,7 +804,7 @@ mapper_registry.map_imperatively(model.WorkflowInvocation, model.WorkflowInvocat input_dataset_collections=relation(model.WorkflowRequestToInputDatasetCollectionAssociation, backref='workflow_invocation'), subworkflow_invocations=relation(model.WorkflowInvocationToSubworkflowInvocationAssociation, - primaryjoin=(model.WorkflowInvocationToSubworkflowInvocationAssociation.table.c.workflow_invocation_id == model.WorkflowInvocation.table.c.id), + primaryjoin=(model.WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id == model.WorkflowInvocation.table.c.id), backref=backref("parent_workflow_invocation", uselist=False), uselist=True, ), @@ -823,15 +815,10 @@ mapper_registry.map_imperatively(model.WorkflowInvocation, model.WorkflowInvocat back_populates='workflow_invocation'), output_datasets=relation(model.WorkflowInvocationOutputDatasetAssociation, back_populates='workflow_invocation'), -)) - -mapper_registry.map_imperatively(model.WorkflowInvocationToSubworkflowInvocationAssociation, model.WorkflowInvocationToSubworkflowInvocationAssociation.table, properties=dict( - subworkflow_invocation=relation(model.WorkflowInvocation, - primaryjoin=(model.WorkflowInvocationToSubworkflowInvocationAssociation.table.c.subworkflow_invocation_id == model.WorkflowInvocation.table.c.id), - backref="parent_workflow_invocation_association", - uselist=False, - ), - workflow_step=relation(model.WorkflowStep), + parent_workflow_invocation_association=relation(model.WorkflowInvocationToSubworkflowInvocationAssociation, + primaryjoin=( + model.WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id == model.WorkflowInvocation.table.c.id), + back_populates='subworkflow_invocation'), )) simple_mapping(model.WorkflowInvocationStep, @@ -839,9 +826,9 @@ simple_mapping(model.WorkflowInvocationStep, job=relation(model.Job, backref=backref('workflow_invocation_step', uselist=False), uselist=False), implicit_collection_jobs=relation(model.ImplicitCollectionJobs, backref=backref('workflow_invocation_step', uselist=False), uselist=False), subworkflow_invocation_id=column_property( - select(model.WorkflowInvocationToSubworkflowInvocationAssociation.table.c.subworkflow_invocation_id).where(and_( - model.WorkflowInvocationToSubworkflowInvocationAssociation.table.c.workflow_invocation_id == model.WorkflowInvocationStep.table.c.workflow_invocation_id, - model.WorkflowInvocationToSubworkflowInvocationAssociation.table.c.workflow_step_id == model.WorkflowInvocationStep.table.c.workflow_step_id, + select(model.WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id).where(and_( + model.WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id == model.WorkflowInvocationStep.table.c.workflow_invocation_id, + model.WorkflowInvocationToSubworkflowInvocationAssociation.workflow_step_id == model.WorkflowInvocationStep.table.c.workflow_step_id, )).scalar_subquery(), ), output_dataset_collections=relation( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 7e442498e75..fd3bbf5fa65 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3979,6 +3979,41 @@ class TestWorkflowInvocationStepOutputDatasetCollectionAssociation(BaseTest): assert stored_obj.dataset_collection.id == history_dataset_collection_association.id +class TestWorkflowInvocationToSubworkflowInvocationAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_invocation_to_subworkflow_invocation_association' + + def test_columns(self, session, cls_, workflow_invocation, workflow_step, model, workflow): + subworkflow_invocation = model.WorkflowInvocation() + subworkflow_invocation.workflow = workflow + persist(session, subworkflow_invocation) + + obj = cls_() + obj.workflow_invocation_id = workflow_invocation.id + obj.subworkflow_invocation = subworkflow_invocation + obj.workflow_step = workflow_step + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.subworkflow_invocation_id == subworkflow_invocation.id + assert stored_obj.workflow_step_id == workflow_step.id + + delete_from_database(session, [subworkflow_invocation]) + + def test_relationships(self, session, cls_, workflow_invocation, workflow_step, model, workflow): + obj = cls_() + obj.subworkflow_invocation = workflow_invocation # We need only 1 instance, so we use the fixture + obj.workflow_step = workflow_step + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.subworkflow_invocation.id == workflow_invocation.id + assert stored_obj.workflow_step.id == workflow_step.id + + class TestWorkflowOutput(BaseTest): def test_table(self, cls_): From d1973c9f9ae5fea62c7893342f545f5462fec518 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 18:07:17 -0400 Subject: [PATCH 156/225] Map StoredWorkflowUserShareAssoc declaratively --- lib/galaxy/model/__init__.py | 19 ++++++++++-- lib/galaxy/model/mapping.py | 32 +++----------------- test/unit/model/test_model.py | 57 +++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 29 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 0471eb6d9bd..fa196f08ced 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6354,14 +6354,29 @@ class WorkflowOutput(Base, RepresentById): return copied_output -class StoredWorkflowUserShareAssociation(UserShareAssociation): +class StoredWorkflowUserShareAssociation(Base, UserShareAssociation): + __tablename__ = 'stored_workflow_user_share_connection' + + id = Column(Integer, primary_key=True) + stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + user = relationship('User', back_populates='workflows_shared_by_others') + stored_workflow = relationship('StoredWorkflow', back_populates='users_shared_with') def __init__(self): self.stored_workflow = None self.user = None -class StoredWorkflowMenuEntry(RepresentById): +class StoredWorkflowMenuEntry(Base, RepresentById): + __tablename__ = 'stored_workflow_menu_entry' + + id = Column(Integer, primary_key=True) + stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + order_index = Column(Integer) + + stored_workflow = relationship('StoredWorkflow') def __init__(self): self.stored_workflow = None diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 70014ebfbfb..76ad0084002 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -354,19 +354,6 @@ model.WorkflowInvocationOutputValue.table = Table( Column("value", MutableJSONType), ) -model.StoredWorkflowUserShareAssociation.table = Table( - "stored_workflow_user_share_connection", metadata, - Column("id", Integer, primary_key=True), - Column("stored_workflow_id", Integer, ForeignKey("stored_workflow.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True)) - -model.StoredWorkflowMenuEntry.table = Table( - "stored_workflow_menu_entry", metadata, - Column("id", Integer, primary_key=True), - Column("stored_workflow_id", Integer, ForeignKey("stored_workflow.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("order_index", Integer)) - model.MetadataFile.table = Table( "metadata_file", metadata, Column("id", Integer, primary_key=True), @@ -535,8 +522,8 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( social_auth=relation(model.UserAuthnzToken, back_populates='user'), stored_workflow_menu_entries=relation(model.StoredWorkflowMenuEntry, primaryjoin=( - (model.StoredWorkflowMenuEntry.table.c.user_id == model.User.table.c.id) - & (model.StoredWorkflowMenuEntry.table.c.stored_workflow_id == model.StoredWorkflow.table.c.id) + (model.StoredWorkflowMenuEntry.user_id == model.User.table.c.id) + & (model.StoredWorkflowMenuEntry.stored_workflow_id == model.StoredWorkflow.table.c.id) & not_(model.StoredWorkflow.table.c.deleted) ), backref="user", @@ -555,6 +542,7 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( histories_shared_by_others=relation( model.HistoryUserShareAssociation, back_populates='user'), data_manager_histories=relation(model.DataManagerHistoryAssociation, back_populates='user'), + workflows_shared_by_others=relation(model.StoredWorkflowUserShareAssociation, back_populates='user'), )) # Set up proxy so that this syntax is possible: @@ -774,7 +762,8 @@ mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.tabl average_rating=column_property( select(func.avg(model.StoredWorkflowRatingAssociation.rating)).where(model.StoredWorkflowRatingAssociation.stored_workflow_id == model.StoredWorkflow.table.c.id).scalar_subquery(), deferred=True - ) + ), + users_shared_with=relation(model.StoredWorkflowUserShareAssociation, back_populates='stored_workflow'), )) # Set up proxy so that @@ -782,17 +771,6 @@ mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.tabl # returns a list of users that workflow is shared with. model.StoredWorkflow.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore -mapper_registry.map_imperatively(model.StoredWorkflowUserShareAssociation, model.StoredWorkflowUserShareAssociation.table, properties=dict( - user=relation(model.User, - backref='workflows_shared_by_others'), - stored_workflow=relation(model.StoredWorkflow, - backref='users_shared_with') -)) - -mapper_registry.map_imperatively(model.StoredWorkflowMenuEntry, model.StoredWorkflowMenuEntry.table, properties=dict( - stored_workflow=relation(model.StoredWorkflow) -)) - mapper_registry.map_imperatively(model.WorkflowInvocation, model.WorkflowInvocation.table, properties=dict( history=relation(model.History, backref=backref('workflow_invocations', uselist=True)), input_parameters=relation(model.WorkflowRequestInputParameter, back_populates='workflow_invocation'), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index fd3bbf5fa65..997502f4dc5 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3189,6 +3189,36 @@ class TestStoredWorkflowAnnotationAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestStoredWorkflowMenuEntry(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'stored_workflow_menu_entry' + + def test_columns(self, session, cls_, stored_workflow, user): + order_index = 1 + obj = cls_() + obj.stored_workflow = stored_workflow + obj.user = user + obj.order_index = order_index + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.stored_workflow_id == stored_workflow.id + assert stored_obj.user_id == user.id + assert stored_obj.order_index == order_index + + def test_relationships(self, session, cls_, stored_workflow, user): + obj = cls_() + obj.stored_workflow = stored_workflow + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.stored_workflow.id == stored_workflow.id + assert stored_obj.user.id == user.id + + class TestStoredWorkflowRatingAssociation(BaseTest): def test_table(self, cls_): @@ -3246,6 +3276,33 @@ class TestStoredWorkflowTagAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestStoredWorkflowUserShareAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'stored_workflow_user_share_connection' + + def test_columns(self, session, cls_, stored_workflow, user): + obj = cls_() + obj.stored_workflow = stored_workflow + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.stored_workflow_id == stored_workflow.id + assert stored_obj.user_id == user.id + + def test_relationships(self, session, cls_, stored_workflow, user): + obj = cls_() + obj.stored_workflow = stored_workflow + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.stored_workflow.id == stored_workflow.id + assert stored_obj.user.id == user.id + + class TestTag(BaseTest): def test_table(self, cls_): From a603d4f52ca2340c16c51d87f07d119f9770e1e4 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 22:02:02 -0400 Subject: [PATCH 157/225] Map DatasetCollectionElement declaratively --- lib/galaxy/model/__init__.py | 22 +++++++++++- lib/galaxy/model/mapping.py | 27 ++------------- test/unit/model/test_model.py | 65 +++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 25 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fa196f08ced..fe2c020f90e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5553,8 +5553,28 @@ class LibraryDatasetCollectionAssociation(DatasetCollectionInstance, RepresentBy return dict_value -class DatasetCollectionElement(Dictifiable, RepresentById): +class DatasetCollectionElement(Base, Dictifiable, RepresentById): """ Associates a DatasetInstance (hda or ldda) with a DatasetCollection. """ + __tablename__ = 'dataset_collection_element' + + id = Column(Integer, primary_key=True) + # Parent collection id describing what collection this element belongs to. + dataset_collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True, nullable=False) + # Child defined by this association - HDA, LDDA, or another dataset association... + hda_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True, nullable=True) + ldda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) + child_collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True, nullable=True) + # Element index and identifier to define this parent-child relationship. + element_index = Column(Integer) + element_identifier = Column(Unicode(255)) + + hda = relationship('HistoryDatasetAssociation', + primaryjoin=(lambda: DatasetCollectionElement.hda_id == HistoryDatasetAssociation.id)) # type: ignore + ldda = relationship('LibraryDatasetDatasetAssociation', + primaryjoin=(lambda: DatasetCollectionElement.ldda_id == LibraryDatasetDatasetAssociation.id)) # type: ignore + child_collection = relationship('DatasetCollection', + primaryjoin=(lambda: DatasetCollectionElement.child_collection_id == DatasetCollection.id)) # type: ignore + # actionable dataset id needs to be available via API... dict_collection_visible_keys = ['id', 'element_type', 'element_index', 'element_identifier'] dict_element_visible_keys = ['id', 'element_type', 'element_index', 'element_identifier'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 76ad0084002..3a5ebd28748 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -274,19 +274,6 @@ model.LibraryDatasetCollectionAssociation.table = Table( Column("name", TrimmedString(255)), Column("deleted", Boolean, default=False)) -model.DatasetCollectionElement.table = Table( - "dataset_collection_element", metadata, - Column("id", Integer, primary_key=True), - # Parent collection id describing what collection this element belongs to. - Column("dataset_collection_id", Integer, ForeignKey("dataset_collection.id"), index=True, nullable=False), - # Child defined by this association - HDA, LDDA, or another dataset association... - Column("hda_id", Integer, ForeignKey("history_dataset_association.id"), index=True, nullable=True), - Column("ldda_id", Integer, ForeignKey("library_dataset_dataset_association.id"), index=True, nullable=True), - Column("child_collection_id", Integer, ForeignKey("dataset_collection.id"), index=True, nullable=True), - # Element index and identifier to define this parent-child relationship. - Column("element_index", Integer), - Column("element_identifier", Unicode(255), )) - model.StoredWorkflow.table = Table( "stored_workflow", metadata, Column("id", Integer, primary_key=True), @@ -639,10 +626,10 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod simple_mapping(model.DatasetCollection, elements=relation(model.DatasetCollectionElement, - primaryjoin=(model.DatasetCollection.table.c.id == model.DatasetCollectionElement.table.c.dataset_collection_id), - remote_side=[model.DatasetCollectionElement.table.c.dataset_collection_id], + primaryjoin=(model.DatasetCollection.table.c.id == model.DatasetCollectionElement.dataset_collection_id), + remote_side=[model.DatasetCollectionElement.dataset_collection_id], backref="collection", - order_by=model.DatasetCollectionElement.table.c.element_index), + order_by=model.DatasetCollectionElement.element_index), output_dataset_collections=relation( model.JobToImplicitOutputDatasetCollectionAssociation, back_populates='dataset_collection'), ) @@ -705,14 +692,6 @@ simple_mapping(model.LibraryDatasetCollectionAssociation, order_by=model.LibraryDatasetCollectionRatingAssociation.id, back_populates="dataset_collection")) -simple_mapping(model.DatasetCollectionElement, - hda=relation(model.HistoryDatasetAssociation, - primaryjoin=(model.DatasetCollectionElement.table.c.hda_id == model.HistoryDatasetAssociation.table.c.id)), - ldda=relation(model.LibraryDatasetDatasetAssociation, - primaryjoin=(model.DatasetCollectionElement.table.c.ldda_id == model.LibraryDatasetDatasetAssociation.table.c.id)), - child_collection=relation(model.DatasetCollection, - primaryjoin=(model.DatasetCollectionElement.table.c.child_collection_id == model.DatasetCollection.table.c.id))) - mapper_registry.map_imperatively(model.Workflow, model.Workflow.table, properties=dict( steps=relation(model.WorkflowStep, backref='workflow', diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 997502f4dc5..43c963b2a93 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -460,6 +460,71 @@ class TestDataset(BaseTest): assert stored_obj.sources == [dataset_source] +class TestDatasetCollectionElement(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'dataset_collection_element' + + def test_columns( + self, + session, + cls_, + dataset_collection, + history_dataset_association, + library_dataset_dataset_association, + model, + ): + element_index, element_identifier = 1, 'a' + obj = cls_(element=history_dataset_association) # using hda is sufficient for this test + obj.element_index = element_index + obj.element_identifier = element_identifier + obj.hda = history_dataset_association + obj.ldda = library_dataset_dataset_association + obj.child_collection = dataset_collection + + # set dataset_collection_id (can't set directly; persisted automatically) + parent_collection = model.DatasetCollection(collection_type='a') + parent_collection.elements.append(obj) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.dataset_collection_id == parent_collection.id + assert stored_obj.hda_id == history_dataset_association.id + assert stored_obj.ldda_id == library_dataset_dataset_association.id + assert stored_obj.child_collection_id == dataset_collection.id + assert stored_obj.element_index == element_index + assert stored_obj.element_identifier == element_identifier + + delete_from_database(session, [parent_collection]) # because this won't be removed automatically + + def test_relationships( + self, + session, + cls_, + dataset_collection, + history_dataset_association, + library_dataset_dataset_association, + model, + ): + obj = cls_(element=history_dataset_association) # using hda is sufficient for this test + obj.hda = history_dataset_association + obj.ldda = library_dataset_dataset_association + obj.child_collection = dataset_collection + + # set dataset_collection_id (can't set directly; persisted automatically) + parent_collection = model.DatasetCollection(collection_type='a') + parent_collection.elements.append(obj) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.hda.id == history_dataset_association.id + assert stored_obj.ldda.id == library_dataset_dataset_association.id + assert stored_obj.child_collection.id == dataset_collection.id + + delete_from_database(session, [parent_collection]) # because this won't be removed automatically + + class TestDatasetHash(BaseTest): def test_table(self, cls_): From 515ed02ee94af0b2c52871fc8cf4e23d93f34df9 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 23:27:47 -0400 Subject: [PATCH 158/225] Map HistoryDatasetAssociationSubset declaratively --- lib/galaxy/model/__init__.py | 16 +++++++++++++++- lib/galaxy/model/mapping.py | 16 ---------------- test/unit/model/test_model.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fe2c020f90e..cc72f41b30b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4318,7 +4318,21 @@ class HistoryDatasetAssociationDisplayAtAuthorization(Base, RepresentById): self.site = site -class HistoryDatasetAssociationSubset(RepresentById): +class HistoryDatasetAssociationSubset(Base, RepresentById): + __tablename__ = 'history_dataset_association_subset' + + id = Column(Integer, primary_key=True) + history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + history_dataset_association_subset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + location = Column(Unicode(255), index=True) + + hda = relationship('HistoryDatasetAssociation', + primaryjoin=(lambda: HistoryDatasetAssociationSubset.history_dataset_association_id # type: ignore + == HistoryDatasetAssociation.id)) # type: ignore + subset = relationship('HistoryDatasetAssociation', + primaryjoin=(lambda: HistoryDatasetAssociationSubset.history_dataset_association_subset_id # type: ignore + == HistoryDatasetAssociation.id)) # type: ignore + def __init__(self, hda, subset, location): self.hda = hda self.subset = subset diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 3a5ebd28748..5ec9e8b81a4 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -124,13 +124,6 @@ model.Dataset.table = Table( Column('total_size', Numeric(15, 0)), Column('uuid', UUIDType())) -model.HistoryDatasetAssociationSubset.table = Table( - "history_dataset_association_subset", metadata, - Column("id", Integer, primary_key=True), - Column("history_dataset_association_id", Integer, ForeignKey("history_dataset_association.id"), index=True), - Column("history_dataset_association_subset_id", Integer, ForeignKey("history_dataset_association.id"), index=True), - Column("location", Unicode(255), index=True)) - model.ImplicitlyConvertedDatasetAssociation.table = Table( "implicitly_converted_dataset_association", metadata, Column("id", Integer, primary_key=True), @@ -454,15 +447,6 @@ simple_mapping(model.Dataset, genome_index_tool_data=relation(model.GenomeIndexToolData, back_populates='dataset'), ) -mapper_registry.map_imperatively(model.HistoryDatasetAssociationSubset, model.HistoryDatasetAssociationSubset.table, properties=dict( - hda=relation(model.HistoryDatasetAssociation, - primaryjoin=(model.HistoryDatasetAssociationSubset.table.c.history_dataset_association_id - == model.HistoryDatasetAssociation.table.c.id)), - subset=relation(model.HistoryDatasetAssociation, - primaryjoin=(model.HistoryDatasetAssociationSubset.table.c.history_dataset_association_subset_id - == model.HistoryDatasetAssociation.table.c.id)) -)) - mapper_registry.map_imperatively(model.ImplicitlyConvertedDatasetAssociation, model.ImplicitlyConvertedDatasetAssociation.table, properties=dict( parent_hda=relation(model.HistoryDatasetAssociation, primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 43c963b2a93..194cae163b5 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1321,6 +1321,41 @@ class TestHistoryDatasetAssociationHistory(BaseTest): assert stored_obj._metadata == metadata +class TestHistoryDatasetAssociationSubset(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_association_subset' + + def test_columns(self, session, cls_, history_dataset_association, model, dataset): + hda_subset = model.HistoryDatasetAssociation(dataset=dataset) + persist(session, hda_subset) + + location = 'a' + obj = cls_(history_dataset_association, hda_subset, location) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.history_dataset_association_subset_id == hda_subset.id + assert stored_obj.location == location + + delete_from_database(session, [hda_subset]) + + def test_relationships(self, session, cls_, history_dataset_association, model, dataset): + hda_subset = model.HistoryDatasetAssociation(dataset=dataset) + persist(session, hda_subset) + + obj = cls_(history_dataset_association, hda_subset, None) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.hda.id == history_dataset_association.id + assert stored_obj.subset.id == hda_subset.id + + delete_from_database(session, [hda_subset]) + + class TestHistoryUserShareAssociation(BaseTest): def test_table(self, cls_): From 79576675cbda16d92b9aaebe78614e6656b61501 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 5 Aug 2021 23:58:54 -0400 Subject: [PATCH 159/225] Map LibraryFolderInfoAssociation declaratively --- lib/galaxy/model/__init__.py | 22 +++++++++++++++- lib/galaxy/model/mapping.py | 22 ---------------- test/unit/model/test_model.py | 48 ++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index cc72f41b30b..7a40c739cf4 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4468,6 +4468,7 @@ class LibraryFolder(Base, Dictifiable, HasName, RepresentById): library_root = relationship('Library', back_populates='root_folder') actions = relationship('LibraryFolderPermissions', back_populates='folder') + info_association = relationship('LibraryFolderInfoAssociation', back_populates='folder') dict_element_visible_keys = ['id', 'parent_id', 'name', 'description', 'item_count', 'genome_build', 'update_time', 'deleted'] @@ -4895,7 +4896,26 @@ class LibraryInfoAssociation(Base, RepresentById): self.inheritable = inheritable -class LibraryFolderInfoAssociation(RepresentById): +class LibraryFolderInfoAssociation(Base, RepresentById): + __tablename__ = 'library_folder_info_association' + + id = Column(Integer, primary_key=True) + library_folder_id = Column(Integer, ForeignKey('library_folder.id'), nullable=True, index=True) + form_definition_id = Column(Integer, ForeignKey('form_definition.id'), index=True) + form_values_id = Column(Integer, ForeignKey('form_values.id'), index=True) + inheritable = Column(Boolean, index=True, default=False) + deleted = Column(Boolean, index=True, default=False) + + folder = relationship('LibraryFolder', + primaryjoin=(lambda: + (LibraryFolderInfoAssociation.library_folder_id == LibraryFolder.id) # type: ignore + & (not_(LibraryFolderInfoAssociation.deleted))), # type: ignore + back_populates="info_association") + template = relationship('FormDefinition', + primaryjoin=(lambda: LibraryFolderInfoAssociation.form_definition_id == FormDefinition.id)) # type: ignore + info = relationship('FormValues', + primaryjoin=(lambda: LibraryFolderInfoAssociation.form_values_id == FormValues.id)) # type: ignore + def __init__(self, folder, form_definition, info, inheritable=False): self.folder = folder self.template = form_definition diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 5ec9e8b81a4..6407f94f158 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -178,15 +178,6 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), Column("message", TrimmedString(255))) -model.LibraryFolderInfoAssociation.table = Table( - "library_folder_info_association", metadata, - Column("id", Integer, primary_key=True), - Column("library_folder_id", Integer, ForeignKey("library_folder.id"), nullable=True, index=True), - Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True), - Column("form_values_id", Integer, ForeignKey("form_values.id"), index=True), - Column("inheritable", Boolean, index=True, default=False), - Column("deleted", Boolean, index=True, default=False)) - model.LibraryDatasetDatasetInfoAssociation.table = Table( "library_dataset_dataset_info_association", metadata, Column("id", Integer, primary_key=True), @@ -534,19 +525,6 @@ mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssoci ) )) -mapper_registry.map_imperatively(model.LibraryFolderInfoAssociation, model.LibraryFolderInfoAssociation.table, properties=dict( - folder=relation(model.LibraryFolder, - primaryjoin=( - (model.LibraryFolderInfoAssociation.table.c.library_folder_id == model.LibraryFolder.id) - & (not_(model.LibraryFolderInfoAssociation.table.c.deleted)) - ), - backref="info_association"), - template=relation(model.FormDefinition, - primaryjoin=(model.LibraryFolderInfoAssociation.table.c.form_definition_id == model.FormDefinition.id)), - info=relation(model.FormValues, - primaryjoin=(model.LibraryFolderInfoAssociation.table.c.form_values_id == model.FormValues.id)) -)) - mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssociation.table, properties=dict( dataset=relation(model.Dataset, primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.dataset_id == model.Dataset.table.c.id), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 194cae163b5..87342ac6bc6 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -2667,13 +2667,23 @@ class TestLibraryFolder(BaseTest): assert stored_obj.genome_build == genome_build def test_relationships( - self, session, cls_, model, library_folder, library_dataset, library, library_folder_permission): + self, + session, + cls_, + model, + library_folder, + library_dataset, + library, + library_folder_permission, + library_folder_info_association, + ): obj = cls_() obj.parent = library_folder folder1 = model.LibraryFolder() obj.folders.append(folder1) obj.library_root.append(library) obj.actions.append(library_folder_permission) + obj.info_association.append(library_folder_info_association) # There's no back reference, so dataset does not update folder; so we have to flush to the database # via dbcleanup() context manager. TODO: ..but why is there no back reference? @@ -2689,6 +2699,36 @@ class TestLibraryFolder(BaseTest): # use identity equality instread of object equality. assert stored_obj.datasets[0].id == library_dataset.id assert stored_obj.active_datasets[0].id == library_dataset.id + assert stored_obj.info_association == [library_folder_info_association] + + +class TestLibraryFolderInfoAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_folder_info_association' + + def test_columns(self, session, cls_, library_folder, form_definition, form_values): + inheritable, deleted = True, True + obj = cls_(library_folder, form_definition, form_values, inheritable) + obj.deleted = deleted + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.library_folder_id == library_folder.id + assert stored_obj.form_definition_id == form_definition.id + assert stored_obj.form_values_id == form_values.id + assert stored_obj.inheritable == inheritable + assert stored_obj.deleted == deleted + + def test_relationships(self, session, cls_, library_folder, form_definition, form_values): + obj = cls_(library_folder, form_definition, form_values) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.folder.id == library_folder.id + assert stored_obj.template.id == form_definition.id + assert stored_obj.info.id == form_values.id class TestLibraryFolderPermissions(BaseTest): @@ -4937,6 +4977,12 @@ def library_dataset_dataset_association_permission(model, session, library_datas yield from dbcleanup_wrapper(session, lddp) +@pytest.fixture +def library_folder_info_association(model, session, library_folder, form_definition, form_values): + lfia = model.LibraryFolderInfoAssociation(library_folder, form_definition, form_values) + yield from dbcleanup_wrapper(session, lfia) + + @pytest.fixture def library_folder_permission(model, session, library_folder, role): lfp = model.LibraryFolderPermissions('a', library_folder, role) From 0d5ce6c971c192c0af45e5b6518deba5f334964e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 6 Aug 2021 01:43:06 -0400 Subject: [PATCH 160/225] Map Visualization declaratively --- lib/galaxy/model/__init__.py | 58 ++++++++++++++++- lib/galaxy/model/mapping.py | 44 ------------- test/unit/model/test_model.py | 115 ++++++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 47 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7a40c739cf4..e4f7e563150 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -7695,7 +7695,50 @@ class PageUserShareAssociation(Base, UserShareAssociation): self.user = None -class Visualization(RepresentById): +class Visualization(Base, RepresentById): + __tablename__ = 'visualization' + __table_args__ = ( + Index('ix_visualization_dbkey', 'dbkey', mysql_length=200), + Index('ix_visualization_slug', 'slug', mysql_length=200), + ) + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=False) + latest_revision_id = Column(Integer, + ForeignKey('visualization_revision.id', use_alter=True, name='visualization_latest_revision_id_fk'), + index=True) + title = Column(TEXT) + type = Column(TEXT) + dbkey = Column(TEXT) + deleted = Column(Boolean, default=False, index=True) + importable = Column(Boolean, default=False, index=True) + slug = Column(TEXT) + published = Column(Boolean, default=False, index=True) + + user = relationship('User') + revisions = relationship('VisualizationRevision', + back_populates='visualization', + cascade="all, delete-orphan", + primaryjoin=(lambda: Visualization.id == VisualizationRevision.visualization_id)) # type: ignore + latest_revision = relationship('VisualizationRevision', + post_update=True, + primaryjoin=(lambda: Visualization.latest_revision_id == VisualizationRevision.id), # type: ignore + lazy=False) + tags = relationship('VisualizationTagAssociation', + order_by='VisualizationTagAssociation.id', + back_populates="visualization") + annotations = relationship('VisualizationAnnotationAssociation', + order_by='VisualizationAnnotationAssociation.id', + back_populates="visualization") + ratings = relationship('VisualizationRatingAssociation', + order_by='VisualizationRatingAssociation.id', + back_populates="visualization") + # `average_rating` added at the bottom of this module + # (cannot be added before VisualizationRatingAssociation is defined) + average_rating: column_property + def __init__(self, id=None, user=None, type=None, title=None, dbkey=None, slug=None, latest_revision=None): self.id = id self.user = user @@ -7704,8 +7747,8 @@ class Visualization(RepresentById): self.dbkey = dbkey self.slug = slug self.latest_revision = latest_revision - self.revisions = [] - if self.latest_revision: + self.revisions = [] # TODO: review: this attempts to overwrite an instrumented attr (SA overwrites this?) + if self.latest_revision: # TODO: but this happens ONLY if passed to constructor. Review. self.revisions.append(latest_revision) def copy(self, user=None, title=None): @@ -7743,6 +7786,9 @@ class VisualizationRevision(Base, RepresentById): title = Column(TEXT) dbkey = Column(TEXT) config = Column(MutableJSONType) + visualization = relationship('Visualization', + back_populates='revisions', + primaryjoin=(lambda: Visualization.id == VisualizationRevision.visualization_id)) # type: ignore def __init__(self, visualization=None, title=None, dbkey=None, config=None): self.id = None @@ -8445,3 +8491,9 @@ History.average_rating = column_property( select(func.avg(HistoryRatingAssociation.rating)).where(HistoryRatingAssociation.history_id == History.id).scalar_subquery(), deferred=True ) + +# Must be defined after VisualizationRatingAssociation has been mapped +Visualization.average_rating = column_property( + select(func.avg(VisualizationRatingAssociation.rating)).where(VisualizationRatingAssociation.visualization_id == Visualization.id).scalar_subquery(), + deferred=True +) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6407f94f158..255cb175e8d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -338,25 +338,6 @@ model.MetadataFile.table = Table( Column("deleted", Boolean, index=True, default=False), Column("purged", Boolean, index=True, default=False)) -model.Visualization.table = Table( - "visualization", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True, nullable=False), - Column("latest_revision_id", Integer, - ForeignKey("visualization_revision.id", use_alter=True, name='visualization_latest_revision_id_fk'), index=True), - Column("title", TEXT), - Column("type", TEXT), - Column("dbkey", TEXT), - Column("deleted", Boolean, default=False, index=True), - Column("importable", Boolean, default=False, index=True), - Column("slug", TEXT), - Column("published", Boolean, default=False, index=True), - Index('ix_visualization_dbkey', 'dbkey', mysql_length=200), - Index('ix_visualization_slug', 'slug', mysql_length=200), -) - model.UserPreference.table = Table( "user_preference", metadata, Column("id", Integer, primary_key=True), @@ -783,31 +764,6 @@ simple_mapping( # returns a list of users that page is shared with. model.Page.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore -mapper_registry.map_imperatively(model.Visualization, model.Visualization.table, properties=dict( - user=relation(model.User), - revisions=relation(model.VisualizationRevision, - backref='visualization', - cascade="all, delete-orphan", - primaryjoin=(model.Visualization.table.c.id == model.VisualizationRevision.visualization_id)), - latest_revision=relation(model.VisualizationRevision, - post_update=True, - primaryjoin=(model.Visualization.table.c.latest_revision_id == model.VisualizationRevision.id), - lazy=False), - tags=relation(model.VisualizationTagAssociation, - order_by=model.VisualizationTagAssociation.id, - back_populates="visualization"), - annotations=relation(model.VisualizationAnnotationAssociation, - order_by=model.VisualizationAnnotationAssociation.id, - back_populates="visualization"), - ratings=relation(model.VisualizationRatingAssociation, - order_by=model.VisualizationRatingAssociation.id, - back_populates="visualization"), - average_rating=column_property( - select(func.avg(model.VisualizationRatingAssociation.rating)).where(model.VisualizationRatingAssociation.visualization_id == model.Visualization.table.c.id).scalar_subquery(), - deferred=True - ) -)) - # Set up proxy so that # Visualization.users_shared_with # returns a list of users that visualization is shared with. diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 87342ac6bc6..450485d4107 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3863,6 +3863,103 @@ class TestUserQuotaAssociation(BaseTest): assert stored_obj.quota.id == quota.id +class TestVisualization(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'visualization' + assert has_index(cls_.__table__, ('dbkey',)) + assert has_index(cls_.__table__, ('slug',)) + + def test_columns(self, session, cls_, user, visualization_revision): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + title = 'a' + type = 'b' + dbkey = 'c' + deleted = True + importable = True + slug = 'd' + published = True + + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.user = user + obj.latest_revision = visualization_revision + obj.title = title + obj.type = type + obj.dbkey = dbkey + obj.deleted = deleted + obj.importable = importable + obj.slug = slug + obj.published = published + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.user_id == user.id + assert stored_obj.latest_revision_id == visualization_revision.id + assert stored_obj.title == title + assert stored_obj.type == type + assert stored_obj.dbkey == dbkey + assert stored_obj.deleted == deleted + assert stored_obj.importable == importable + assert stored_obj.slug == slug + assert stored_obj.published == published + + def test_relationships( + self, + session, + cls_, + user, + visualization_revision, + model, + visualization, + visualization_tag_association, + visualization_annotation_association, + visualization_rating_association, + ): + revision2 = model.VisualizationRevision(visualization=visualization) + persist(session, revision2) + + obj = cls_() + obj.user = user + obj.latest_revision = visualization_revision + obj.revisions.append(revision2) + obj.tags.append(visualization_tag_association) + obj.annotations.append(visualization_annotation_association) + obj.ratings.append(visualization_rating_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id + assert stored_obj.latest_revision.id == visualization_revision.id + assert stored_obj.revisions == [revision2] + assert stored_obj.tags == [visualization_tag_association] + assert stored_obj.annotations == [visualization_annotation_association] + assert stored_obj.ratings == [visualization_rating_association] + # This doesn't test the average amount, just the mapping. + assert stored_obj.average_rating == visualization_rating_association.rating + + def test_average_rating(self, model, session, visualization, user): + # Visualization has been expunged; to access its deferred properties, + # it needs to be added back to the session. + session.add(visualization) + assert visualization.average_rating is None # With no ratings, we expect None. + # Create ratings + to_cleanup = [] + for rating in (1, 2, 3, 4, 5): + visualization_rating_assoc = model.VisualizationRatingAssociation(user, visualization) + visualization_rating_assoc.rating = rating + persist(session, visualization_rating_assoc) + to_cleanup.append(visualization_rating_assoc) + assert visualization.average_rating == 3.0 # Expect average after ratings added. + # Cleanup: remove ratings from database + delete_from_database(session, to_cleanup) + + class TestVisualizationAnnotationAssociation(BaseTest): def test_table(self, cls_): @@ -5137,6 +5234,24 @@ def visualization(model, session, user): yield from dbcleanup_wrapper(session, v) +@pytest.fixture +def visualization_annotation_association(model, session): + vaa = model.VisualizationAnnotationAssociation() + yield from dbcleanup_wrapper(session, vaa) + + +@pytest.fixture +def visualization_rating_association(model, session, user, visualization): + vaa = model.VisualizationRatingAssociation(user, visualization) + yield from dbcleanup_wrapper(session, vaa) + + +@pytest.fixture +def visualization_revision(model, session, visualization): + vr = model.VisualizationRevision(visualization=visualization) + yield from dbcleanup_wrapper(session, vr) + + @pytest.fixture def visualization_tag_association(model, session): vta = model.VisualizationTagAssociation() From 8919a4be74179a6bedc2c0ab73ef61194ec044c8 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 7 Aug 2021 16:18:30 -0400 Subject: [PATCH 161/225] Add fixture factories In some tests we may need more than one instance of the same model. We cannot reuse a model fixture, and we cannot pass multiple copies of the same fixture to one test. We have to instantiate a new instance of the model inside the test. However, a test should only know how to construct the model it is testing, so instead of constructing an object directly, a test calls a factory function, passed to it as a fixture. --- test/unit/model/test_model.py | 255 ++++++++++++++++++++++++++++------ 1 file changed, 211 insertions(+), 44 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 450485d4107..e53bae9a973 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -435,7 +435,7 @@ class TestDataset(BaseTest): library_dataset_dataset_association, dataset_hash, dataset_source, - model, + history_dataset_association_factory, ): obj = cls_() obj.job = job @@ -445,7 +445,7 @@ class TestDataset(BaseTest): obj.hashes.append(dataset_hash) obj.sources.append(dataset_source) - hda = model.HistoryDatasetAssociation() + hda = history_dataset_association_factory() hda.purged = True obj.history_associations.append(hda) @@ -472,7 +472,7 @@ class TestDatasetCollectionElement(BaseTest): dataset_collection, history_dataset_association, library_dataset_dataset_association, - model, + dataset_collection_factory, ): element_index, element_identifier = 1, 'a' obj = cls_(element=history_dataset_association) # using hda is sufficient for this test @@ -483,7 +483,7 @@ class TestDatasetCollectionElement(BaseTest): obj.child_collection = dataset_collection # set dataset_collection_id (can't set directly; persisted automatically) - parent_collection = model.DatasetCollection(collection_type='a') + parent_collection = dataset_collection_factory() parent_collection.elements.append(obj) with dbcleanup(session, obj) as obj_id: @@ -505,7 +505,7 @@ class TestDatasetCollectionElement(BaseTest): dataset_collection, history_dataset_association, library_dataset_dataset_association, - model, + dataset_collection_factory, ): obj = cls_(element=history_dataset_association) # using hda is sufficient for this test obj.hda = history_dataset_association @@ -513,7 +513,7 @@ class TestDatasetCollectionElement(BaseTest): obj.child_collection = dataset_collection # set dataset_collection_id (can't set directly; persisted automatically) - parent_collection = model.DatasetCollection(collection_type='a') + parent_collection = dataset_collection_factory() parent_collection.elements.append(obj) with dbcleanup(session, obj) as obj_id: @@ -1238,7 +1238,6 @@ class TestHistory(BaseTest): default_history_permissions, history_user_share_association, galaxy_session_history_association, - model, ): obj = cls_() obj.user = user @@ -1274,7 +1273,7 @@ class TestHistory(BaseTest): assert stored_obj.default_permissions == [default_history_permissions] assert stored_obj.galaxy_sessions == [galaxy_session_history_association] - def test_average_rating(self, model, session, history, user): + def test_average_rating(self, session, history, user, history_rating_association_factory): # History has been expunged; to access its deferred properties, # it needs to be added back to the session. session.add(history) @@ -1282,8 +1281,7 @@ class TestHistory(BaseTest): # Create ratings to_cleanup = [] for rating in (1, 2, 3, 4, 5): - history_rating_assoc = model.HistoryRatingAssociation(user, history) - history_rating_assoc.rating = rating + history_rating_assoc = history_rating_association_factory(user, history, rating) persist(session, history_rating_assoc) to_cleanup.append(history_rating_assoc) assert history.average_rating == 3.0 # Expect average after ratings added. @@ -1326,8 +1324,15 @@ class TestHistoryDatasetAssociationSubset(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'history_dataset_association_subset' - def test_columns(self, session, cls_, history_dataset_association, model, dataset): - hda_subset = model.HistoryDatasetAssociation(dataset=dataset) + def test_columns( + self, + session, + cls_, + history_dataset_association, + history, + history_dataset_association_factory, + ): + hda_subset = history_dataset_association_factory() persist(session, hda_subset) location = 'a' @@ -1342,8 +1347,14 @@ class TestHistoryDatasetAssociationSubset(BaseTest): delete_from_database(session, [hda_subset]) - def test_relationships(self, session, cls_, history_dataset_association, model, dataset): - hda_subset = model.HistoryDatasetAssociation(dataset=dataset) + def test_relationships( + self, + session, + cls_, + history_dataset_association, + history_dataset_association_factory, + ): + hda_subset = history_dataset_association_factory() persist(session, hda_subset) obj = cls_(history_dataset_association, hda_subset, None) @@ -1754,10 +1765,16 @@ class TestImplicitlyCreatedDatasetCollectionInput(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'implicitly_created_dataset_collection_inputs' - def test_columns(self, session, cls_, history_dataset_collection_association, model): + def test_columns( + self, + session, + cls_, + history_dataset_collection_association, + history_dataset_collection_association_factory, + ): name = 'a' - hdca2 = model.HistoryDatasetCollectionAssociation() + hdca2 = history_dataset_collection_association_factory() persist(session, hdca2) obj = cls_(name, history_dataset_collection_association) @@ -1772,8 +1789,14 @@ class TestImplicitlyCreatedDatasetCollectionInput(BaseTest): delete_from_database(session, [hdca2]) - def test_relationships(self, session, cls_, history_dataset_collection_association, model): - hdca2 = model.HistoryDatasetCollectionAssociation() + def test_relationships( + self, + session, + cls_, + history_dataset_collection_association, + history_dataset_collection_association_factory, + ): + hdca2 = history_dataset_collection_association_factory() persist(session, hdca2) obj = cls_(None, history_dataset_collection_association) @@ -2348,14 +2371,14 @@ class TestLibraryDataset(BaseTest): library_dataset_dataset_association, library_folder, library_dataset_permission, - model, + library_dataset_dataset_association_factory, ): obj = cls_() obj.folder = library_folder obj.library_dataset_dataset_association = library_dataset_dataset_association obj.actions.append(library_dataset_permission) - ldda = model.LibraryDatasetDatasetAssociation() + ldda = library_dataset_dataset_association_factory() ldda.library_dataset = obj obj.actions.append(library_dataset_permission) @@ -2670,16 +2693,16 @@ class TestLibraryFolder(BaseTest): self, session, cls_, - model, library_folder, library_dataset, library, library_folder_permission, library_folder_info_association, + library_folder_factory, ): obj = cls_() obj.parent = library_folder - folder1 = model.LibraryFolder() + folder1 = library_folder_factory() obj.folders.append(folder1) obj.library_root.append(library) obj.actions.append(library_folder_permission) @@ -2890,7 +2913,7 @@ class TestPage(BaseTest): # This doesn't test the average amount, just the mapping. assert stored_obj.average_rating == page_rating_association.rating - def test_average_rating(self, model, session, page, user): + def test_average_rating(self, session, page, user, page_rating_association_factory): # Page has been expunged; to access its deferred properties, # it needs to be added back to the session. session.add(page) @@ -2898,8 +2921,7 @@ class TestPage(BaseTest): # Create ratings to_cleanup = [] for rating in (1, 2, 3, 4, 5): - page_rating_assoc = model.PageRatingAssociation(user, page) - page_rating_assoc.rating = rating + page_rating_assoc = page_rating_association_factory(user, page, rating) persist(session, page_rating_assoc) to_cleanup.append(page_rating_assoc) assert page.average_rating == 3.0 # Expect average after ratings added. @@ -3915,13 +3937,12 @@ class TestVisualization(BaseTest): cls_, user, visualization_revision, - model, - visualization, visualization_tag_association, visualization_annotation_association, visualization_rating_association, + visualization_revision_factory, ): - revision2 = model.VisualizationRevision(visualization=visualization) + revision2 = visualization_revision_factory() persist(session, revision2) obj = cls_() @@ -3943,7 +3964,7 @@ class TestVisualization(BaseTest): # This doesn't test the average amount, just the mapping. assert stored_obj.average_rating == visualization_rating_association.rating - def test_average_rating(self, model, session, visualization, user): + def test_average_rating(self, session, visualization, user, visualization_rating_association_factory): # Visualization has been expunged; to access its deferred properties, # it needs to be added back to the session. session.add(visualization) @@ -3951,8 +3972,7 @@ class TestVisualization(BaseTest): # Create ratings to_cleanup = [] for rating in (1, 2, 3, 4, 5): - visualization_rating_assoc = model.VisualizationRatingAssociation(user, visualization) - visualization_rating_assoc.rating = rating + visualization_rating_assoc = visualization_rating_association_factory(user, visualization, rating) persist(session, visualization_rating_assoc) to_cleanup.append(visualization_rating_assoc) assert visualization.average_rating == 3.0 # Expect average after ratings added. @@ -4278,8 +4298,16 @@ class TestWorkflowInvocationToSubworkflowInvocationAssociation(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'workflow_invocation_to_subworkflow_invocation_association' - def test_columns(self, session, cls_, workflow_invocation, workflow_step, model, workflow): - subworkflow_invocation = model.WorkflowInvocation() + def test_columns( + self, + session, + cls_, + workflow_invocation, + workflow_step, + workflow, + workflow_invocation_factory, + ): + subworkflow_invocation = workflow_invocation_factory() subworkflow_invocation.workflow = workflow persist(session, subworkflow_invocation) @@ -4297,7 +4325,14 @@ class TestWorkflowInvocationToSubworkflowInvocationAssociation(BaseTest): delete_from_database(session, [subworkflow_invocation]) - def test_relationships(self, session, cls_, workflow_invocation, workflow_step, model, workflow): + def test_relationships( + self, + session, + cls_, + workflow_invocation, + workflow_step, + workflow, + ): obj = cls_() obj.subworkflow_invocation = workflow_invocation # We need only 1 instance, so we use the fixture obj.workflow_step = workflow_step @@ -4475,7 +4510,7 @@ class TestWorkflowStep(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'workflow_step' - def test_columns(self, session, cls_, workflow, dynamic_tool, model): + def test_columns(self, session, cls_, workflow, dynamic_tool, workflow_factory): create_time = datetime.now() update_time = create_time + timedelta(hours=1) type = 'a' @@ -4488,7 +4523,7 @@ class TestWorkflowStep(BaseTest): order_index = 'h' label = 'k' - subworkflow = model.Workflow() + subworkflow = workflow_factory() persist(session, subworkflow) obj = cls_() @@ -4528,10 +4563,18 @@ class TestWorkflowStep(BaseTest): delete_from_database(session, [subworkflow]) - def test_relationships(self, session, cls_, workflow, dynamic_tool, model): - subworkflow = model.Workflow() - workflow_step_connection_in = model.WorkflowStepConnection() - workflow_step_connection_out = model.WorkflowStepConnection() + def test_relationships( + self, + session, + cls_, + workflow, + dynamic_tool, + workflow_factory, + workflow_step_connection_factory, + ): + subworkflow = workflow_factory() + workflow_step_connection_in = workflow_step_connection_factory() + workflow_step_connection_out = workflow_step_connection_factory() extra_objects = [subworkflow, workflow_step_connection_in, workflow_step_connection_out] for eo in extra_objects: persist(session, eo) @@ -4590,10 +4633,18 @@ class TestWorkflowStepConnection(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'workflow_step_connection' - def test_columns(self, session, cls_, workflow_step_input, workflow_step, workflow, model): + def test_columns( + self, + session, + cls_, + workflow_step_input, + workflow_step, + workflow, + workflow_step_factory, + ): output_name = 'a' - output_workflow_step = model.WorkflowStep() + output_workflow_step = workflow_step_factory() output_workflow_step.workflow = workflow persist(session, output_workflow_step) @@ -4613,8 +4664,16 @@ class TestWorkflowStepConnection(BaseTest): delete_from_database(session, [output_workflow_step]) - def test_relationships(self, session, cls_, workflow_step_input, workflow_step, workflow, model): - output_workflow_step = model.WorkflowStep() + def test_relationships( + self, + session, + cls_, + workflow_step_input, + workflow_step, + workflow, + workflow_step_factory, + ): + output_workflow_step = workflow_step_factory() output_workflow_step.workflow = workflow persist(session, output_workflow_step) @@ -4772,6 +4831,8 @@ class TestWorkflowRequestInputParameter(BaseTest): assert stored_obj.workflow_invocation.id == workflow_invocation.id +# Misc. helper fixtures. + @pytest.fixture(scope='module') def model(): db_uri = 'sqlite:///:memory:' @@ -4785,6 +4846,8 @@ def session(model): Session.remove() # Ensures we get a new session for each test +# Fixtures yielding persisted instances of models, deleted from the database on test exit. + @pytest.fixture def cleanup_event(model, session): ce = model.CleanupEvent() @@ -5310,6 +5373,110 @@ def workflow_step_tag_association(model, session): yield from dbcleanup_wrapper(session, wsta) +# Fixtures yielding factory functions. +# In some tests we may need more than one instance of the same model. We cannot reuse a model +# fixture, and we cannot pass multiple copies of the same fixture to one test. We have to +# instantiate a new instance of the model inside the test. However, a test should only know +# how to construct the model it is testing, so instead of constructing an object directly, +# a test calls a factory function, passed to it as a fixture. + +@pytest.fixture +def dataset_collection_factory(model): + def make_instance(*args, **kwds): + if 'collection_type' not in kwds: + kwds['collection_type'] = 'a' + return model.DatasetCollection(*args, **kwds) + return make_instance + + +@pytest.fixture +def history_dataset_association_factory(model): + def make_instance(*args, **kwds): + return model.HistoryDatasetAssociation(*args, **kwds) + return make_instance + + +@pytest.fixture +def history_dataset_collection_association_factory(model): + def make_instance(*args, **kwds): + return model.HistoryDatasetCollectionAssociation(*args, **kwds) + return make_instance + + +@pytest.fixture +def history_rating_association_factory(model, user, history): + def make_instance(*args, **kwds): + return model.HistoryRatingAssociation(*args, **kwds) + return make_instance + + +@pytest.fixture +def library_dataset_dataset_association_factory(model): + def make_instance(*args, **kwds): + return model.LibraryDatasetDatasetAssociation(*args, **kwds) + return make_instance + + +@pytest.fixture +def library_folder_factory(model): + def make_instance(*args, **kwds): + return model.LibraryFolder(*args, **kwds) + return make_instance + + +@pytest.fixture +def page_rating_association_factory(model): + def make_instance(*args, **kwds): + return model.PageRatingAssociation(*args, **kwds) + return make_instance + + +@pytest.fixture +def visualization_rating_association_factory(model): + def make_instance(*args, **kwds): + return model.VisualizationRatingAssociation(*args, **kwds) + return make_instance + + +@pytest.fixture +def visualization_revision_factory(model, visualization): + def make_instance(*args, **kwds): + if 'visualization' not in kwds: + kwds['visualization'] = visualization + return model.VisualizationRevision(*args, **kwds) + return make_instance + + +@pytest.fixture +def workflow_factory(model): + def make_instance(*args, **kwds): + return model.Workflow(*args, **kwds) + return make_instance + + +@pytest.fixture +def workflow_invocation_factory(model): + def make_instance(*args, **kwds): + return model.WorkflowInvocation(*args, **kwds) + return make_instance + + +@pytest.fixture +def workflow_step_connection_factory(model): + def make_instance(*args, **kwds): + return model.WorkflowStepConnection(*args, **kwds) + return make_instance + + +@pytest.fixture +def workflow_step_factory(model): + def make_instance(*args, **kwds): + return model.WorkflowStep(*args, **kwds) + return make_instance + + +# Mics. helpers. + def dbcleanup_wrapper(session, obj): with dbcleanup(session, obj): yield obj From f1c2b62606fefd224fd30031f54b7a63635a5f0c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 9 Aug 2021 14:52:54 -0400 Subject: [PATCH 162/225] Map DatasetCollection declaratively (see note) The remote_side attr in the `elements` relationship is not needed here: the join condition is specified in primaryjoin (and it's not a self-referential relationship). --- lib/galaxy/model/__init__.py | 25 +++++++++++++-- lib/galaxy/model/mapping.py | 20 ------------ test/unit/model/test_model.py | 59 +++++++++++++++++++++++++++++++++-- 3 files changed, 79 insertions(+), 25 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e4f7e563150..6206a28c623 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4974,9 +4974,24 @@ class ImplicitlyConvertedDatasetAssociation(RepresentById): DEFAULT_COLLECTION_NAME = "Unnamed Collection" -class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById): - """ - """ +class DatasetCollection(Base, Dictifiable, UsesAnnotations, RepresentById): + __tablename__ = 'dataset_collection' + + id = Column(Integer, primary_key=True) + collection_type = Column(Unicode(255), nullable=False) + populated_state = Column(TrimmedString(64), default='ok', nullable=False) + populated_state_message = Column(TEXT) + element_count = Column(Integer, nullable=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + + elements = relationship('DatasetCollectionElement', + primaryjoin=(lambda: DatasetCollection.id == DatasetCollectionElement.dataset_collection_id), # type: ignore + back_populates='collection', + order_by='DatasetCollectionElement.element_index') + output_dataset_collections = relationship( + 'JobToImplicitOutputDatasetCollectionAssociation', back_populates='dataset_collection') + dict_collection_visible_keys = ['id', 'collection_type'] dict_element_visible_keys = ['id', 'collection_type'] @@ -5608,6 +5623,10 @@ class DatasetCollectionElement(Base, Dictifiable, RepresentById): primaryjoin=(lambda: DatasetCollectionElement.ldda_id == LibraryDatasetDatasetAssociation.id)) # type: ignore child_collection = relationship('DatasetCollection', primaryjoin=(lambda: DatasetCollectionElement.child_collection_id == DatasetCollection.id)) # type: ignore + collection = relationship('DatasetCollection', + primaryjoin=(lambda: DatasetCollection.id == DatasetCollectionElement.dataset_collection_id), # type: ignore + back_populates='elements', + ) # actionable dataset id needs to be available via API... dict_collection_visible_keys = ['id', 'element_type', 'element_index', 'element_identifier'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 255cb175e8d..838a604823a 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -223,16 +223,6 @@ model.Job.table = Table( Column("params", TrimmedString(255), index=True), Column("handler", TrimmedString(255), index=True)) -model.DatasetCollection.table = Table( - "dataset_collection", metadata, - Column("id", Integer, primary_key=True), - Column("collection_type", Unicode(255), nullable=False), - Column("populated_state", TrimmedString(64), default='ok', nullable=False), - Column("populated_state_message", TEXT), - Column("element_count", Integer, nullable=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now)) - model.HistoryDatasetCollectionAssociation.table = Table( "history_dataset_collection_association", metadata, Column("id", Integer, primary_key=True), @@ -567,16 +557,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod # ), # ) -simple_mapping(model.DatasetCollection, - elements=relation(model.DatasetCollectionElement, - primaryjoin=(model.DatasetCollection.table.c.id == model.DatasetCollectionElement.dataset_collection_id), - remote_side=[model.DatasetCollectionElement.dataset_collection_id], - backref="collection", - order_by=model.DatasetCollectionElement.element_index), - output_dataset_collections=relation( - model.JobToImplicitOutputDatasetCollectionAssociation, back_populates='dataset_collection'), -) - simple_mapping(model.HistoryDatasetCollectionAssociation, collection=relation(model.DatasetCollection), history=relation(model.History, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index e53bae9a973..cc2069f947b 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -460,6 +460,55 @@ class TestDataset(BaseTest): assert stored_obj.sources == [dataset_source] +class TestDatasetCollection(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'dataset_collection' + + def test_columns(self, session, cls_): + collection_type = 'a' + populated_state = 'b' + populated_state_message = 'c' + element_count = 1 + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + + obj = cls_() + obj.collection_type = collection_type + obj.populated_state = populated_state + obj.populated_state_message = populated_state_message + obj.element_count = element_count + obj.create_time = create_time + obj.update_time = update_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.collection_type == collection_type + assert stored_obj.populated_state == populated_state + assert stored_obj.populated_state_message == populated_state_message + assert stored_obj.element_count == element_count + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + + def test_relationships( + self, + session, + cls_, + dataset_collection_element, + job_to_implicit_output_dataset_collection_association, + ): + obj = cls_() + obj.collection_type, obj.populated_state = 'a', 'b' + obj.elements.append(dataset_collection_element) + obj.output_dataset_collections.append(job_to_implicit_output_dataset_collection_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.elements == [dataset_collection_element] + assert stored_obj.output_dataset_collections == [job_to_implicit_output_dataset_collection_association] + + class TestDatasetCollectionElement(BaseTest): def test_table(self, cls_): @@ -512,15 +561,15 @@ class TestDatasetCollectionElement(BaseTest): obj.ldda = library_dataset_dataset_association obj.child_collection = dataset_collection - # set dataset_collection_id (can't set directly; persisted automatically) parent_collection = dataset_collection_factory() - parent_collection.elements.append(obj) + obj.collection = parent_collection # same as parent_collection.elements.append(obj) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.hda.id == history_dataset_association.id assert stored_obj.ldda.id == library_dataset_dataset_association.id assert stored_obj.child_collection.id == dataset_collection.id + assert stored_obj.collection.id == parent_collection.id delete_from_database(session, [parent_collection]) # because this won't be removed automatically @@ -5083,6 +5132,12 @@ def job_export_history_archive(model, session): yield from dbcleanup_wrapper(session, jeha) +@pytest.fixture +def job_to_implicit_output_dataset_collection_association(model, session, dataset_collection): + instance = model.JobToImplicitOutputDatasetCollectionAssociation(None, dataset_collection) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def library(model, session): lb = model.Library() From e2bdcb68d23734d387bcdbd07b901dad041cd140 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 9 Aug 2021 22:28:55 -0400 Subject: [PATCH 163/225] Map HistoryDatasetCollectionAssoc declaratively --- lib/galaxy/model/__init__.py | 94 +++++++++++++++++++-- lib/galaxy/model/mapping.py | 71 ++-------------- test/unit/model/test_model.py | 149 ++++++++++++++++++++++++++++++++-- 3 files changed, 238 insertions(+), 76 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6206a28c623..e5945f44350 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -90,6 +90,7 @@ from galaxy.model.custom_types import ( ) from galaxy.model.item_attrs import get_item_annotation_str, UsesAnnotations from galaxy.model.orm.now import now +from galaxy.model.view import HistoryDatasetCollectionJobStateSummary from galaxy.security import get_permitted_actions from galaxy.security.validate_user_input import validate_password_str from galaxy.util import ( @@ -1792,6 +1793,10 @@ class ImplicitlyCreatedDatasetCollectionInput(Base, RepresentById): primaryjoin=( lambda: HistoryDatasetCollectionAssociation.id == ImplicitlyCreatedDatasetCollectionInput.input_dataset_collection_id # type: ignore )) + dataset_collection = relationship('HistoryDatasetCollectionAssociation', + primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id # type: ignore + == ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id), # type: ignore + back_populates='implicit_input_collections') def __init__(self, name, input_dataset_collection): self.name = name @@ -1804,6 +1809,8 @@ class ImplicitCollectionJobs(Base, RepresentById): id = Column(Integer, primary_key=True) populated_state = Column(TrimmedString(64), default='new', nullable=False) jobs = relationship('ImplicitCollectionJobsJobAssociation', back_populates='implicit_collection_jobs') + history_dataset_collection_associations = relationship('HistoryDatasetCollectionAssociation', + back_populates='implicit_collection_jobs') class populated_states(str, Enum): NEW = 'new' # New implicit jobs object, unpopulated job associations @@ -2300,6 +2307,7 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, RepresentByI ), order_by=lambda: asc(HistoryDatasetAssociation.hid), # type: ignore viewonly=True) + dataset_collections = relationship('HistoryDatasetCollectionAssociation', back_populates='history') active_dataset_collections = relationship('HistoryDatasetCollectionAssociation', primaryjoin=( lambda: (and_(HistoryDatasetCollectionAssociation.history_id == History.id, # type: ignore @@ -5361,12 +5369,88 @@ class DatasetCollectionInstance(HasName): return changed -class HistoryDatasetCollectionAssociation(DatasetCollectionInstance, - HasTags, - Dictifiable, - UsesAnnotations, - RepresentById): +class HistoryDatasetCollectionAssociation( + Base, + DatasetCollectionInstance, + HasTags, + Dictifiable, + UsesAnnotations, + RepresentById +): """ Associates a DatasetCollection with a History. """ + __tablename__ = 'history_dataset_collection_association' + + id = Column(Integer, primary_key=True) + collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + name = Column(TrimmedString(255)) + hid = Column(Integer) + visible = Column(Boolean) + deleted = Column(Boolean, default=False) + copied_from_history_dataset_collection_association_id = Column(Integer, + ForeignKey('history_dataset_collection_association.id'), nullable=True) + implicit_output_name = Column(Unicode(255), nullable=True) + job_id = Column(ForeignKey('job.id'), index=True, nullable=True) + implicit_collection_jobs_id = Column( + ForeignKey('implicit_collection_jobs.id'), index=True, nullable=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now, index=True) + + collection = relationship('DatasetCollection') + history = relationship('History', back_populates='dataset_collections') + + copied_from_history_dataset_collection_association = relationship( + 'HistoryDatasetCollectionAssociation', + primaryjoin=copied_from_history_dataset_collection_association_id == id, + remote_side=[id], + uselist=False, + back_populates='copied_to_history_dataset_collection_association', + ) + copied_to_history_dataset_collection_association = relationship( + 'HistoryDatasetCollectionAssociation', + back_populates='copied_from_history_dataset_collection_association', + ) + implicit_input_collections = relationship('ImplicitlyCreatedDatasetCollectionInput', + primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id # type: ignore + == ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id), # type: ignore + back_populates='dataset_collection', + ) + implicit_collection_jobs = relationship( + 'ImplicitCollectionJobs', + back_populates='history_dataset_collection_associations', + uselist=False + ) + job = relationship( + 'Job', + back_populates='history_dataset_collection_associations', + uselist=False, + ) + job_state_summary = relationship(HistoryDatasetCollectionJobStateSummary, + primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id # type: ignore + == HistoryDatasetCollectionJobStateSummary.__table__.c.hdca_id), # type: ignore + foreign_keys=HistoryDatasetCollectionJobStateSummary.__table__.c.hdca_id, + uselist=False, + ) + tags = relationship( + 'HistoryDatasetCollectionTagAssociation', + order_by='HistoryDatasetCollectionTagAssociation.id', + back_populates='dataset_collection', + ) + annotations = relationship( + 'HistoryDatasetCollectionAssociationAnnotationAssociation', + order_by='HistoryDatasetCollectionAssociationAnnotationAssociation.id', + back_populates='history_dataset_collection', + ) + ratings = relationship( + 'HistoryDatasetCollectionRatingAssociation', + order_by='HistoryDatasetCollectionRatingAssociation.id', + back_populates='dataset_collection', + ) + output_dataset_collection_instances = relationship( + 'JobToOutputDatasetCollectionAssociation', + back_populates='dataset_collection_instance', + ) + editable_keys = ('name', 'deleted', 'visible') def __init__( diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 838a604823a..49888740a52 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -47,7 +47,6 @@ from galaxy.model.migrate.triggers.update_audit_table import install as install_ from galaxy.model.orm.engine_factory import build_engine from galaxy.model.orm.now import now from galaxy.model.security import GalaxyRBACAgent -from galaxy.model.view import HistoryDatasetCollectionJobStateSummary from galaxy.model.view.utils import install_views log = logging.getLogger(__name__) @@ -223,23 +222,6 @@ model.Job.table = Table( Column("params", TrimmedString(255), index=True), Column("handler", TrimmedString(255), index=True)) -model.HistoryDatasetCollectionAssociation.table = Table( - "history_dataset_collection_association", metadata, - Column("id", Integer, primary_key=True), - Column("collection_id", Integer, ForeignKey("dataset_collection.id"), index=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("name", TrimmedString(255)), - Column("hid", Integer), - Column("visible", Boolean), - Column("deleted", Boolean, default=False), - Column("copied_from_history_dataset_collection_association_id", Integer, - ForeignKey("history_dataset_collection_association.id"), nullable=True), - Column("implicit_output_name", Unicode(255), nullable=True), - Column("job_id", ForeignKey("job.id"), index=True, nullable=True), - Column("implicit_collection_jobs_id", ForeignKey("implicit_collection_jobs.id"), index=True, nullable=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now, index=True)) - model.LibraryDatasetCollectionAssociation.table = Table( "library_dataset_collection_association", metadata, Column("id", Integer, primary_key=True), @@ -374,7 +356,7 @@ simple_mapping(model.HistoryDatasetAssociation, == model.ExtendedMetadata.id)), hidden_beneath_collection_instance=relation(model.HistoryDatasetCollectionAssociation, primaryjoin=(model.HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id - == model.HistoryDatasetCollectionAssociation.table.c.id), + == model.HistoryDatasetCollectionAssociation.id), uselist=False, backref="hidden_dataset_instances"), _metadata=deferred(model.HistoryDatasetAssociation.table.c._metadata), @@ -557,50 +539,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod # ), # ) -simple_mapping(model.HistoryDatasetCollectionAssociation, - collection=relation(model.DatasetCollection), - history=relation(model.History, - backref='dataset_collections'), - copied_from_history_dataset_collection_association=relation(model.HistoryDatasetCollectionAssociation, - primaryjoin=(model.HistoryDatasetCollectionAssociation.table.c.copied_from_history_dataset_collection_association_id - == model.HistoryDatasetCollectionAssociation.table.c.id), - remote_side=[model.HistoryDatasetCollectionAssociation.table.c.id], - backref='copied_to_history_dataset_collection_associations', - uselist=False), - implicit_input_collections=relation(model.ImplicitlyCreatedDatasetCollectionInput, - primaryjoin=(model.HistoryDatasetCollectionAssociation.table.c.id - == model.ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id), - backref="dataset_collection", - ), - implicit_collection_jobs=relation( - model.ImplicitCollectionJobs, - backref=backref("history_dataset_collection_associations", uselist=True), - uselist=False, - ), - job=relation( - model.Job, - backref=backref("history_dataset_collection_associations", uselist=True), - uselist=False, - ), - job_state_summary=relation(HistoryDatasetCollectionJobStateSummary, - primaryjoin=(model.HistoryDatasetCollectionAssociation.table.c.id == HistoryDatasetCollectionJobStateSummary.__table__.c.hdca_id), - foreign_keys=HistoryDatasetCollectionJobStateSummary.__table__.c.hdca_id, - uselist=False - ), - tags=relation(model.HistoryDatasetCollectionTagAssociation, - order_by=model.HistoryDatasetCollectionTagAssociation.id, - back_populates='dataset_collection'), - annotations=relation(model.HistoryDatasetCollectionAssociationAnnotationAssociation, - order_by=model.HistoryDatasetCollectionAssociationAnnotationAssociation.id, - back_populates="history_dataset_collection"), - ratings=relation(model.HistoryDatasetCollectionRatingAssociation, - order_by=model.HistoryDatasetCollectionRatingAssociation.id, - back_populates="dataset_collection"), - output_dataset_collection_instances=relation( - model.JobToOutputDatasetCollectionAssociation, - back_populates='dataset_collection_instance'), -) - simple_mapping(model.LibraryDatasetCollectionAssociation, collection=relation(model.DatasetCollection), folder=relation(model.LibraryFolder, @@ -780,6 +718,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( back_populates='job', uselist=False), container=relation('JobContainerAssociation', back_populates='job', uselist=False), data_manager_association=relation('DataManagerJobAssociation', back_populates='job', uselist=False), + history_dataset_collection_associations=relation('HistoryDatasetCollectionAssociation', back_populates='job'), )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], @@ -789,10 +728,10 @@ model.Job.any_output_dataset_deleted = column_property( # type: ignore ) ) model.Job.any_output_dataset_collection_instances_deleted = column_property( # type: ignore - exists([model.HistoryDatasetCollectionAssociation.table.c.id], + exists([model.HistoryDatasetCollectionAssociation.id], and_(model.Job.table.c.id == model.JobToOutputDatasetCollectionAssociation.job_id, - model.HistoryDatasetCollectionAssociation.table.c.id == model.JobToOutputDatasetCollectionAssociation.dataset_collection_id, - model.HistoryDatasetCollectionAssociation.table.c.deleted == true()) + model.HistoryDatasetCollectionAssociation.id == model.JobToOutputDatasetCollectionAssociation.dataset_collection_id, + model.HistoryDatasetCollectionAssociation.deleted == true()) ) ) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index cc2069f947b..ad5f5928e9d 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1609,6 +1609,108 @@ class TestHistoryDatasetAssociationTagAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestHistoryDatasetCollectionAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_collection_association' + + def test_columns( + self, + session, + cls_, + dataset_collection, + history, + history_dataset_collection_association, + job, + implicit_collection_jobs, + ): + name = 'a' + hid = 1 + visible = True + deleted = True + implicit_output_name = 'b' + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + + obj = cls_() + obj.collection = dataset_collection + obj.history = history + obj.name = name + obj.hid = hid + obj.visible = visible + obj.deleted = deleted + obj.copied_from_history_dataset_collection_association = history_dataset_collection_association + obj.implicit_output_name = implicit_output_name + obj.job = job + obj.implicit_collection_jobs = implicit_collection_jobs + obj.create_time = create_time + obj.update_time = update_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.collection_id == dataset_collection.id + assert stored_obj.history_id == history.id + assert stored_obj.name == name + assert stored_obj.hid == hid + assert stored_obj.visible == visible + assert stored_obj.deleted == deleted + assert (stored_obj.copied_from_history_dataset_collection_association_id + == history_dataset_collection_association.id) + assert stored_obj.implicit_output_name == implicit_output_name + assert stored_obj.job_id == job.id + assert stored_obj.implicit_collection_jobs_id == implicit_collection_jobs.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + + def test_relationships( + self, + session, + cls_, + dataset_collection, + history, + history_dataset_collection_association, + history_dataset_collection_association_factory, + job, + implicit_collection_jobs, + implicitly_created_dataset_collection_input, + history_dataset_collection_annotation_association, + history_dataset_collection_rating_association, + history_dataset_collection_tag_association, + job_to_output_dataset_collection_association, + ): + copied_to_hdca = history_dataset_collection_association_factory() + + obj = cls_() + obj.collection = dataset_collection + obj.history = history + obj.copied_from_history_dataset_collection_association = history_dataset_collection_association + obj.copied_to_history_dataset_collection_association.append(copied_to_hdca) + obj.job = job + obj.implicit_collection_jobs = implicit_collection_jobs + obj.implicit_input_collections.append(implicitly_created_dataset_collection_input) + obj.tags.append(history_dataset_collection_tag_association) + obj.annotations.append(history_dataset_collection_annotation_association) + obj.ratings.append(history_dataset_collection_rating_association) + obj.output_dataset_collection_instances.append(job_to_output_dataset_collection_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.collection.id == dataset_collection.id + assert stored_obj.history.id == history.id + assert (stored_obj.copied_from_history_dataset_collection_association.id + == history_dataset_collection_association.id) + assert stored_obj.copied_to_history_dataset_collection_association == [copied_to_hdca] + assert stored_obj.job.id == job.id + assert stored_obj.implicit_collection_jobs.id == implicit_collection_jobs.id + assert stored_obj.implicit_input_collections == [implicitly_created_dataset_collection_input] + assert stored_obj.tags == [history_dataset_collection_tag_association] + assert stored_obj.annotations == [history_dataset_collection_annotation_association] + assert stored_obj.ratings == [history_dataset_collection_rating_association] + assert stored_obj.output_dataset_collection_instances == [job_to_output_dataset_collection_association] + assert stored_obj.job_state_summary # this is a view; TODO: can we test this better? + + class TestHistoryDatasetCollectionAssociationAnnotationAssociation(BaseTest): def test_table(self, cls_): @@ -1767,13 +1869,21 @@ class TestImplicitCollectionJobs(BaseTest): assert stored_obj.id == obj_id assert stored_obj.populated_state == populated_state - def test_relationships(self, session, cls_, implicit_collection_jobs_job_association): + def test_relationships( + self, + session, + cls_, + implicit_collection_jobs_job_association, + history_dataset_collection_association, + ): obj = cls_() obj.jobs.append(implicit_collection_jobs_job_association) + obj.history_dataset_collection_associations.append(history_dataset_collection_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.jobs == [implicit_collection_jobs_job_association] + assert stored_obj.history_dataset_collection_associations == [history_dataset_collection_association] class TestImplicitCollectionJobsJobAssociation(BaseTest): @@ -5070,14 +5180,31 @@ def history_dataset_association_tag_association(model, session): @pytest.fixture def history_dataset_collection_association(model, session): - hdca = model.HistoryDatasetCollectionAssociation() - yield from dbcleanup_wrapper(session, hdca) + instance = model.HistoryDatasetCollectionAssociation() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def history_dataset_collection_annotation_association(model, session): + instance = model.HistoryDatasetCollectionAssociationAnnotationAssociation() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def history_dataset_collection_rating_association( + model, + session, + user, + history_dataset_collection_association, +): + instance = model.HistoryDatasetCollectionRatingAssociation(user, history_dataset_collection_association) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def history_dataset_collection_tag_association(model, session): - hdcta = model.HistoryDatasetCollectionTagAssociation() - yield from dbcleanup_wrapper(session, hdcta) + instance = model.HistoryDatasetCollectionTagAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -5120,6 +5247,12 @@ def implicitly_converted_dataset_association(model, session, history_dataset_ass yield from dbcleanup_wrapper(session, icda) +@pytest.fixture +def implicitly_created_dataset_collection_input(model, session, history_dataset_collection_association): + instance = model.ImplicitlyCreatedDatasetCollectionInput(None, history_dataset_collection_association) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def job(model, session): j = model.Job() @@ -5138,6 +5271,12 @@ def job_to_implicit_output_dataset_collection_association(model, session, datase yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def job_to_output_dataset_collection_association(model, session, history_dataset_collection_association): + instance = model.JobToOutputDatasetCollectionAssociation(None, history_dataset_collection_association) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def library(model, session): lb = model.Library() From f43b6f432780d4f7d7a652dce77dd2839c637884 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 10 Aug 2021 00:35:55 -0400 Subject: [PATCH 164/225] Map LibraryDatasetCollectionAssoc declaratively --- lib/galaxy/model/__init__.py | 24 +++++++++++- lib/galaxy/model/mapping.py | 22 ----------- test/unit/model/test_model.py | 69 +++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e5945f44350..85991cb5d6c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4474,6 +4474,7 @@ class LibraryFolder(Base, Dictifiable, HasName, RepresentById): lazy=True, viewonly=True) + dataset_collections = relationship('LibraryDatasetCollectionAssociation', back_populates='folder') library_root = relationship('Library', back_populates='root_folder') actions = relationship('LibraryFolderPermissions', back_populates='folder') info_association = relationship('LibraryFolderInfoAssociation', back_populates='folder') @@ -5658,8 +5659,29 @@ class HistoryDatasetCollectionAssociation( return len(results) > 0 -class LibraryDatasetCollectionAssociation(DatasetCollectionInstance, RepresentById): +class LibraryDatasetCollectionAssociation(Base, DatasetCollectionInstance, RepresentById): """ Associates a DatasetCollection with a library folder. """ + __tablename__ = 'library_dataset_collection_association' + + id = Column(Integer, primary_key=True) + collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True) + folder_id = Column(Integer, ForeignKey('library_folder.id'), index=True) + name = Column(TrimmedString(255)) + deleted = Column(Boolean, default=False) + + collection = relationship('DatasetCollection') + folder = relationship('LibraryFolder', back_populates='dataset_collections') + + tags = relationship('LibraryDatasetCollectionTagAssociation', + order_by='LibraryDatasetCollectionTagAssociation.id', + back_populates='dataset_collection') + annotations = relationship('LibraryDatasetCollectionAnnotationAssociation', + order_by='LibraryDatasetCollectionAnnotationAssociation.id', + back_populates="dataset_collection") + ratings = relationship('LibraryDatasetCollectionRatingAssociation', + order_by='LibraryDatasetCollectionRatingAssociation.id', + back_populates="dataset_collection") + editable_keys = ('name', 'deleted') def __init__( diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 49888740a52..0379caaabf4 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -222,14 +222,6 @@ model.Job.table = Table( Column("params", TrimmedString(255), index=True), Column("handler", TrimmedString(255), index=True)) -model.LibraryDatasetCollectionAssociation.table = Table( - "library_dataset_collection_association", metadata, - Column("id", Integer, primary_key=True), - Column("collection_id", Integer, ForeignKey("dataset_collection.id"), index=True), - Column("folder_id", Integer, ForeignKey("library_folder.id"), index=True), - Column("name", TrimmedString(255)), - Column("deleted", Boolean, default=False)) - model.StoredWorkflow.table = Table( "stored_workflow", metadata, Column("id", Integer, primary_key=True), @@ -539,20 +531,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod # ), # ) -simple_mapping(model.LibraryDatasetCollectionAssociation, - collection=relation(model.DatasetCollection), - folder=relation(model.LibraryFolder, - backref='dataset_collections'), - tags=relation(model.LibraryDatasetCollectionTagAssociation, - order_by=model.LibraryDatasetCollectionTagAssociation.id, - back_populates='dataset_collection'), - annotations=relation(model.LibraryDatasetCollectionAnnotationAssociation, - order_by=model.LibraryDatasetCollectionAnnotationAssociation.id, - back_populates="dataset_collection"), - ratings=relation(model.LibraryDatasetCollectionRatingAssociation, - order_by=model.LibraryDatasetCollectionRatingAssociation.id, - back_populates="dataset_collection")) - mapper_registry.map_imperatively(model.Workflow, model.Workflow.table, properties=dict( steps=relation(model.WorkflowStep, backref='workflow', diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index ad5f5928e9d..17bff77fa1a 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -2549,6 +2549,54 @@ class TestLibraryDataset(BaseTest): assert stored_obj.actions == [library_dataset_permission] +class TestLibraryDatasetCollectionAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset_collection_association' + + def test_columns(self, session, cls_, dataset_collection, library_folder): + name, deleted = 'a', True + obj = cls_() + obj.collection = dataset_collection + obj.folder = library_folder + obj.name = name + obj.deleted = deleted + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.collection_id == dataset_collection.id + assert stored_obj.folder_id == library_folder.id + assert stored_obj.name == name + assert stored_obj.deleted == deleted + + def test_relationships( + self, + session, + cls_, + dataset_collection, + library_folder, + library_dataset_collection_annotation_association, + library_dataset_collection_rating_association, + library_dataset_collection_tag_association, + ): + + obj = cls_() + obj.collection = dataset_collection + obj.folder = library_folder + obj.tags.append(library_dataset_collection_tag_association) + obj.annotations.append(library_dataset_collection_annotation_association) + obj.ratings.append(library_dataset_collection_rating_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.collection.id == dataset_collection.id + assert stored_obj.folder.id == library_folder.id + assert stored_obj.tags == [library_dataset_collection_tag_association] + assert stored_obj.annotations == [library_dataset_collection_annotation_association] + assert stored_obj.ratings == [library_dataset_collection_rating_association] + + class TestJobToOutputLibraryDatasetAssociation(BaseTest): def test_table(self, cls_): @@ -2855,6 +2903,7 @@ class TestLibraryFolder(BaseTest): library_folder, library_dataset, library, + library_dataset_collection_association, library_folder_permission, library_folder_info_association, library_folder_factory, @@ -2863,6 +2912,7 @@ class TestLibraryFolder(BaseTest): obj.parent = library_folder folder1 = library_folder_factory() obj.folders.append(folder1) + obj.dataset_collections.append(library_dataset_collection_association) obj.library_root.append(library) obj.actions.append(library_folder_permission) obj.info_association.append(library_folder_info_association) @@ -2881,6 +2931,7 @@ class TestLibraryFolder(BaseTest): # use identity equality instread of object equality. assert stored_obj.datasets[0].id == library_dataset.id assert stored_obj.active_datasets[0].id == library_dataset.id + assert stored_obj.dataset_collections == [library_dataset_collection_association] assert stored_obj.info_association == [library_folder_info_association] @@ -5295,12 +5346,30 @@ def library_dataset(model, session): yield from dbcleanup_wrapper(session, ld) +@pytest.fixture +def library_dataset_annotation_association(model, session): + instance = model.LibraryDatasetAnnotationAssociation() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def library_dataset_collection_annotation_association(model, session): + instance = model.LibraryDatasetCollectionAnnotationAssociation() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def library_dataset_collection_association(model, session): ldca = model.LibraryDatasetCollectionAssociation() yield from dbcleanup_wrapper(session, ldca) +@pytest.fixture +def library_dataset_collection_rating_association(model, session): + instance = model.LibraryDatasetCollectionRatingAssociation(None, None) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def library_dataset_collection_tag_association(model, session): ldcta = model.LibraryDatasetCollectionTagAssociation() From 259e1afe3cc2d682bb7a5fd474e676f0a4fa72f3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 10 Aug 2021 11:28:29 -0400 Subject: [PATCH 165/225] Map MetadataFile declaratively --- lib/galaxy/model/__init__.py | 17 +++++++++- lib/galaxy/model/mapping.py | 18 ---------- test/unit/model/test_model.py | 62 +++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 85991cb5d6c..bc2b9893657 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -7189,7 +7189,22 @@ class WorkflowInvocationStepOutputDatasetCollectionAssociation(Base, Dictifiable dict_collection_visible_keys = ['id', 'workflow_invocation_step_id', 'dataset_collection_id', 'output_name'] -class MetadataFile(StorableObject, RepresentById): +class MetadataFile(Base, StorableObject, RepresentById): + __tablename__ = 'metadata_file' + + id = Column(Integer, primary_key=True) + name = Column(TEXT) + hda_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True, nullable=True) + lda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, index=True, default=now, onupdate=now) + object_store_id = Column(TrimmedString(255), index=True) + uuid = Column(UUIDType(), index=True) + deleted = Column(Boolean, index=True, default=False) + purged = Column(Boolean, index=True, default=False) + + history_dataset = relationship('HistoryDatasetAssociation') + library_dataset = relationship('LibraryDatasetDatasetAssociation') def __init__(self, dataset=None, name=None, uuid=None): super().__init__(id=None, uuid=uuid) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 0379caaabf4..a252e2706b7 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -289,19 +289,6 @@ model.WorkflowInvocationOutputValue.table = Table( Column("value", MutableJSONType), ) -model.MetadataFile.table = Table( - "metadata_file", metadata, - Column("id", Integer, primary_key=True), - Column("name", TEXT), - Column("hda_id", Integer, ForeignKey("history_dataset_association.id"), index=True, nullable=True), - Column("lda_id", Integer, ForeignKey("library_dataset_dataset_association.id"), index=True, nullable=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, index=True, default=now, onupdate=now), - Column("object_store_id", TrimmedString(255), index=True), - Column("uuid", UUIDType(), index=True), - Column("deleted", Boolean, index=True, default=False), - Column("purged", Boolean, index=True, default=False)) - model.UserPreference.table = Table( "user_preference", metadata, Column("id", Integer, primary_key=True), @@ -634,11 +621,6 @@ simple_mapping(model.WorkflowInvocationStep, back_populates='workflow_invocation_step'), ) -mapper_registry.map_imperatively(model.MetadataFile, model.MetadataFile.table, properties=dict( - history_dataset=relation(model.HistoryDatasetAssociation), - library_dataset=relation(model.LibraryDatasetDatasetAssociation) -)) - simple_mapping( model.WorkflowInvocationOutputValue, workflow_invocation=relation(model.WorkflowInvocation, backref="output_values"), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 17bff77fa1a..cdffdef341f 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -45,6 +45,7 @@ TestFoo(BaseTest): # BaseTest is parent class import random from contextlib import contextmanager from datetime import datetime, timedelta +from uuid import uuid4 import pytest from sqlalchemy import ( @@ -3055,6 +3056,67 @@ class TestLibraryPermissions(BaseTest): assert stored_obj.role.id == role.id +class TestMetadataFile(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'metadata_file' + + def test_columns( + self, + session, + cls_, + history_dataset_association, + library_dataset_dataset_association, + ): + name = 'a' + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + object_store_id = 'b' + uuid = uuid4() + deleted = True + purged = True + + obj = cls_() + obj.name = name + obj.history_dataset = history_dataset_association + obj.library_dataset = library_dataset_dataset_association + obj.create_time = create_time + obj.update_time = update_time + obj.object_store_id = object_store_id + obj.uuid = uuid + obj.deleted = deleted + obj.purged = purged + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.name == name + assert stored_obj.hda_id == history_dataset_association.id + assert stored_obj.lda_id == library_dataset_dataset_association.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.object_store_id == object_store_id + assert stored_obj.uuid == uuid + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + + def test_relationships( + self, + session, + cls_, + history_dataset_association, + library_dataset_dataset_association, + ): + obj = cls_() + obj.history_dataset = history_dataset_association + obj.library_dataset = library_dataset_dataset_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history_dataset.id == history_dataset_association.id + assert stored_obj.library_dataset.id == library_dataset_dataset_association.id + + class TestPage(BaseTest): def test_table(self, cls_): From bc0aef8ba7e51b95d1db9dc9d0abf5fe0a8799a8 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 10 Aug 2021 15:59:36 -0400 Subject: [PATCH 166/225] Map UserRoleAssociation declaratively --- lib/galaxy/model/__init__.py | 25 +++++++++++++++++++++- lib/galaxy/model/mapping.py | 23 +-------------------- test/unit/model/test_model.py | 39 +++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index bc2b9893657..61a8f070f48 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2763,7 +2763,29 @@ class HistoryUserShareAssociation(Base, UserShareAssociation): self.user = None -class UserRoleAssociation(RepresentById): +class UserRoleAssociation(Base, RepresentById): + __tablename__ = 'user_role_association' + + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + role_id = Column(Integer, ForeignKey('role.id'), index=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + + user = relationship('User', back_populates="roles") + role = relationship('Role', back_populates="users") + + # TODO: this should be defined on the User model only + non_private_roles = relationship( + 'User', + backref="non_private_roles", + viewonly=True, + primaryjoin=(lambda: + (User.table.c.id == UserRoleAssociation.user_id) # type: ignore + & (UserRoleAssociation.role_id == Role.id) # type: ignore + & not_(Role.name == User.table.c.email)) # type: ignore + ) + def __init__(self, user, role): self.user = user self.role = role @@ -2802,6 +2824,7 @@ class Role(Base, Dictifiable, RepresentById): library_dataset_actions = relationship('LibraryDatasetPermissions', back_populates='role') library_dataset_dataset_actions = relationship( 'LibraryDatasetDatasetAssociationPermissions', back_populates='role') + users = relationship('UserRoleAssociation', back_populates='role') dict_collection_visible_keys = ['id', 'name'] dict_element_visible_keys = ['id', 'name', 'description', 'type'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index a252e2706b7..11e5ade6600 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -136,14 +136,6 @@ model.ImplicitlyConvertedDatasetAssociation.table = Table( Column("metadata_safe", Boolean, index=True, default=True), Column("type", TrimmedString(255))) -model.UserRoleAssociation.table = Table( - "user_role_association", metadata, - Column("id", Integer, primary_key=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("role_id", Integer, ForeignKey("role.id"), index=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now)) - model.LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_association", metadata, Column("id", Integer, primary_key=True), @@ -437,26 +429,13 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( model.HistoryUserShareAssociation, back_populates='user'), data_manager_histories=relation(model.DataManagerHistoryAssociation, back_populates='user'), workflows_shared_by_others=relation(model.StoredWorkflowUserShareAssociation, back_populates='user'), + roles=relation(model.UserRoleAssociation, back_populates='user'), )) # Set up proxy so that this syntax is possible: # .preferences[pref_name] = pref_value model.User.preferences = association_proxy('_preferences', 'value', creator=model.UserPreference) # type: ignore -mapper_registry.map_imperatively(model.UserRoleAssociation, model.UserRoleAssociation.table, properties=dict( - user=relation(model.User, backref="roles"), - role=relation(model.Role, backref="users"), - non_private_roles=relation( - model.User, - backref="non_private_roles", - viewonly=True, - primaryjoin=( - (model.User.table.c.id == model.UserRoleAssociation.table.c.user_id) - & (model.UserRoleAssociation.table.c.role_id == model.Role.id) - & not_(model.Role.name == model.User.table.c.email)) - ) -)) - mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssociation.table, properties=dict( dataset=relation(model.Dataset, primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.dataset_id == model.Dataset.table.c.id), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index cdffdef341f..3de4bb7579a 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3572,6 +3572,7 @@ class TestRole(BaseTest): library_folder_permission, library_dataset_permission, library_dataset_dataset_association_permission, + user_role_association, ): name, description, type_ = get_random_string(), 'b', cls_.types.SYSTEM obj = cls_(name, description, type_) @@ -3581,6 +3582,7 @@ class TestRole(BaseTest): obj.library_dataset_actions.append(library_dataset_permission) obj.library_dataset_dataset_actions.append(library_dataset_dataset_association_permission) obj.groups.append(group_role_association) + obj.users.append(user_role_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -3590,6 +3592,7 @@ class TestRole(BaseTest): assert stored_obj.library_folder_actions == [library_folder_permission] assert stored_obj.library_dataset_actions == [library_dataset_permission] assert stored_obj.library_dataset_dataset_actions == [library_dataset_dataset_association_permission] + assert stored_obj.users == [user_role_association] class TestStoredWorkflowAnnotationAssociation(BaseTest): @@ -4157,6 +4160,36 @@ class TestUserQuotaAssociation(BaseTest): assert stored_obj.quota.id == quota.id +class TestUserRoleAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'user_role_association' + + def test_columns(self, session, cls_, user, role): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj = cls_(user, role) + obj.create_time = create_time + obj.update_time = update_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.user_id == user.id + assert stored_obj.role_id == role.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + + def test_relationships(self, session, cls_, user, role): + obj = cls_(user, role) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id + assert stored_obj.role.id == role.id + # TODO non_private_roles should be defined on the User only (test to be added to TestUser) + + class TestVisualization(BaseTest): def test_table(self, cls_): @@ -5615,6 +5648,12 @@ def user_quota_association(model, session): yield from dbcleanup_wrapper(session, uqa) +@pytest.fixture +def user_role_association(model, session, user, role): + instance = model.UserRoleAssociation(user, role) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def visualization(model, session, user): v = model.Visualization() From 4150e9f19201b2262a5e619b74ceef73bffccdb1 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 10 Aug 2021 16:08:08 -0400 Subject: [PATCH 167/225] Map UserPreferences declaratively --- lib/galaxy/model/__init__.py | 9 ++++++++- lib/galaxy/model/mapping.py | 11 ----------- test/unit/model/test_model.py | 26 ++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 61a8f070f48..d00f57d12cc 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -8491,7 +8491,14 @@ class DataManagerJobAssociation(Base, RepresentById): self.data_manager_id = data_manager_id -class UserPreference(RepresentById): +class UserPreference(Base, RepresentById): + __tablename__ = 'user_preference' + + id = Column(Integer, primary_key=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) + name = Column(Unicode(255), index=True) + value = Column(Text) + def __init__(self, name=None, value=None): self.name = name self.value = value diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 11e5ade6600..75efdc4eb5c 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -24,9 +24,7 @@ from sqlalchemy import ( select, String, Table, TEXT, - Text, true, - Unicode, ) from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.orderinglist import ordering_list @@ -281,13 +279,6 @@ model.WorkflowInvocationOutputValue.table = Table( Column("value", MutableJSONType), ) -model.UserPreference.table = Table( - "user_preference", metadata, - Column("id", Integer, primary_key=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("name", Unicode(255), index=True), - Column("value", Text)) - # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -295,8 +286,6 @@ def simple_mapping(model, **kwds): mapper_registry.map_imperatively(model, model.table, properties=kwds) -mapper_registry.map_imperatively(model.UserPreference, model.UserPreference.table, properties={}) - simple_mapping(model.HistoryDatasetAssociation, dataset=relation(model.Dataset, primaryjoin=(model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 3de4bb7579a..e8fd20870cb 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -4131,6 +4131,32 @@ class TestUserGroupAssociation(BaseTest): assert stored_obj.group.id == group.id +class TestUserPreference(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'user_preference' + + def test_columns(self, session, cls_, user): + name, value = 'a', 'b' + obj = cls_(name, value) + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.name == name + assert stored_obj.value == value + assert stored_obj.user_id == user.id + + def test_relationships(self, session, cls_, user): + obj = cls_() + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.user.id == user.id + + class TestUserQuotaAssociation(BaseTest): def test_table(self, cls_): From 991de3a0edf9b697609ab06e0fdc017532d58dbf Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 00:29:26 -0400 Subject: [PATCH 168/225] Add unit tests for WorkflowInvocationStep (see note) Use separate commit to test subworkflow_invocation_id column_property which requires some setup. --- test/unit/model/test_model.py | 137 +++++++++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 2 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index e8fd20870cb..ca83b2c9a78 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -4447,6 +4447,118 @@ class TestWorkerProcess(BaseTest): assert stored_obj.update_time == update_time +class TestWorkflowInvocationStep(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'workflow_invocation_step' + + def test_columns( + self, + session, + cls_, + workflow_invocation, + workflow_step, + job, + implicit_collection_jobs, + ): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + state, action = 'a', 'b' + + session.add(job) # needed for a lazy load of an attr + session.add(implicit_collection_jobs) # needed for a lazy load of an attr + + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.state = state + obj.job = job + obj.implicit_collection_jobs = implicit_collection_jobs + obj.action = action + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.state == state + assert stored_obj.job_id == job.id + assert stored_obj.implicit_collection_jobs_id == implicit_collection_jobs.id + assert stored_obj.action == action + + def test_relationships( + self, + session, + cls_, + workflow_invocation, + workflow_step, + job, + implicit_collection_jobs, + workflow_invocation_step_output_dataset_collection_association, + workflow_invocation_step_output_dataset_association, + ): + session.add(job) # needed for a lazy load of an attr + session.add(implicit_collection_jobs) # needed for a lazy load of an attr + + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.job = job + obj.implicit_collection_jobs = implicit_collection_jobs + obj.output_dataset_collections.append(workflow_invocation_step_output_dataset_collection_association) + obj.output_datasets.append(workflow_invocation_step_output_dataset_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.job.id == job.id + assert stored_obj.implicit_collection_jobs.id == implicit_collection_jobs.id + assert stored_obj.output_dataset_collections == [workflow_invocation_step_output_dataset_collection_association] + assert stored_obj.output_datasets == [workflow_invocation_step_output_dataset_association] + + def test_subworkflow_invocation_attribute( + self, + session, + cls_, + workflow_step, + workflow_invocation_to_subworkflow_invocation_association_factory, + workflow_invocation_factory, + ): + # use defaults to create 2 workflows + workflow_invocation1 = workflow_invocation_factory() + workflow_invocation2 = workflow_invocation_factory() # this is the subworkflow invocation + + # store to retrieve object ids + persist(session, workflow_invocation1) + persist(session, workflow_invocation2) + persist(session, workflow_step) + + # setup assoc object and store + assoc = workflow_invocation_to_subworkflow_invocation_association_factory() + assoc.workflow_invocation_id = workflow_invocation1.id + assoc.subworkflow_invocation_id = workflow_invocation2.id + assoc.workflow_step_id = workflow_step.id + persist(session, assoc) + + # setup main object under test + obj = cls_() + obj.workflow_invocation = workflow_invocation1 + obj.workflow_step = workflow_step + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.subworkflow_invocation_id == workflow_invocation2.id + + # finish cleanup + persisted = [workflow_invocation1, workflow_invocation2, workflow_step, assoc] + delete_from_database(session, persisted) + + class TestWorkflowInvocationOutputDatasetAssociation(BaseTest): def test_table(self, cls_): @@ -5732,6 +5844,18 @@ def workflow_invocation_step(model, session, workflow_invocation, workflow_step) yield from dbcleanup_wrapper(session, wis) +@pytest.fixture +def workflow_invocation_step_output_dataset_collection_association(model, session): + instance = model.WorkflowInvocationStepOutputDatasetCollectionAssociation() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def workflow_invocation_step_output_dataset_association(model, session): + instance = model.WorkflowInvocationStepOutputDatasetAssociation() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def workflow_output(model, session, workflow_step): wo = model.WorkflowOutput(workflow_step) @@ -5845,9 +5969,18 @@ def workflow_factory(model): @pytest.fixture -def workflow_invocation_factory(model): +def workflow_invocation_factory(model, workflow): + def make_instance(**kwds): + instance = model.WorkflowInvocation() + instance.workflow = kwds.get('workflow', workflow) + return instance + return make_instance + + +@pytest.fixture +def workflow_invocation_to_subworkflow_invocation_association_factory(model): def make_instance(*args, **kwds): - return model.WorkflowInvocation(*args, **kwds) + return model.WorkflowInvocationToSubworkflowInvocationAssociation(*args, **kwds) return make_instance From 4f78668590f847724890431fbb38fd765413ffeb Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 01:02:12 -0400 Subject: [PATCH 169/225] Map WorkflowInvocationStep declaratively --- lib/galaxy/model/__init__.py | 38 ++++++++++++++++++++++++++++++++++- lib/galaxy/model/mapping.py | 36 ++++----------------------------- test/unit/model/test_model.py | 7 +++++-- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index d00f57d12cc..27ddab1dfc7 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1811,6 +1811,8 @@ class ImplicitCollectionJobs(Base, RepresentById): jobs = relationship('ImplicitCollectionJobsJobAssociation', back_populates='implicit_collection_jobs') history_dataset_collection_associations = relationship('HistoryDatasetCollectionAssociation', back_populates='implicit_collection_jobs') + workflow_invocation_step = relationship('WorkflowInvocationStep', + back_populates='implicit_collection_jobs', uselist=False) class populated_states(str, Enum): NEW = 'new' # New implicit jobs object, unpopulated job associations @@ -6960,7 +6962,33 @@ class WorkflowInvocationToSubworkflowInvocationAssociation(Base, Dictifiable, Re dict_element_visible_keys = ['id', 'workflow_step_id', 'workflow_invocation_id', 'subworkflow_invocation_id'] -class WorkflowInvocationStep(Dictifiable, RepresentById): +class WorkflowInvocationStep(Base, Dictifiable, RepresentById): + __tablename__ = 'workflow_invocation_step' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + workflow_invocation_id = Column(Integer, + ForeignKey('workflow_invocation.id'), index=True, nullable=False) + workflow_step_id = Column(Integer, + ForeignKey('workflow_step.id'), index=True, nullable=False) + state = Column(TrimmedString(64), index=True) + job_id = Column(Integer, ForeignKey('job.id'), index=True, nullable=True) + implicit_collection_jobs_id = Column(Integer, + ForeignKey('implicit_collection_jobs.id'), index=True, nullable=True) + action = Column(MutableJSONType, nullable=True) + + workflow_step = relationship('WorkflowStep') + job = relationship('Job', back_populates='workflow_invocation_step', uselist=False) + implicit_collection_jobs = relationship('ImplicitCollectionJobs', back_populates='workflow_invocation_step', uselist=False) + output_dataset_collections = relationship( + 'WorkflowInvocationStepOutputDatasetCollectionAssociation', + back_populates='workflow_invocation_step') + output_datasets = relationship('WorkflowInvocationStepOutputDatasetAssociation', + back_populates='workflow_invocation_step') + + subworkflow_invocation_id: column_property + dict_collection_visible_keys = ['id', 'update_time', 'job_id', 'workflow_step_id', 'subworkflow_invocation_id', 'state', 'action'] dict_element_visible_keys = ['id', 'update_time', 'job_id', 'workflow_step_id', 'subworkflow_invocation_id', 'state', 'action'] @@ -8667,3 +8695,11 @@ Visualization.average_rating = column_property( select(func.avg(VisualizationRatingAssociation.rating)).where(VisualizationRatingAssociation.visualization_id == Visualization.id).scalar_subquery(), deferred=True ) + +# Must be defined after WorkflowInvocationToSubworkflowInvocationAssociation +WorkflowInvocationStep.subworkflow_invocation_id = column_property( + select(WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id).where(and_( + WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id == WorkflowInvocationStep.workflow_invocation_id, + WorkflowInvocationToSubworkflowInvocationAssociation.workflow_step_id == WorkflowInvocationStep.workflow_step_id, + )).scalar_subquery(), +) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 75efdc4eb5c..affd048a3b5 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -258,18 +258,6 @@ model.WorkflowInvocation.table = Table( Column('uuid', UUIDType()), Column("history_id", Integer, ForeignKey("history.id"), index=True)) -model.WorkflowInvocationStep.table = Table( - "workflow_invocation_step", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True, nullable=False), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id"), index=True, nullable=False), - Column("state", TrimmedString(64), index=True), - Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=True), - Column("implicit_collection_jobs_id", Integer, ForeignKey("implicit_collection_jobs.id"), index=True, nullable=True), - Column("action", MutableJSONType, nullable=True)) - model.WorkflowInvocationOutputValue.table = Table( "workflow_invocation_output_value", metadata, Column("id", Integer, primary_key=True), @@ -572,31 +560,14 @@ mapper_registry.map_imperatively(model.WorkflowInvocation, model.WorkflowInvocat back_populates='subworkflow_invocation'), )) -simple_mapping(model.WorkflowInvocationStep, - workflow_step=relation(model.WorkflowStep), - job=relation(model.Job, backref=backref('workflow_invocation_step', uselist=False), uselist=False), - implicit_collection_jobs=relation(model.ImplicitCollectionJobs, backref=backref('workflow_invocation_step', uselist=False), uselist=False), - subworkflow_invocation_id=column_property( - select(model.WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id).where(and_( - model.WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id == model.WorkflowInvocationStep.table.c.workflow_invocation_id, - model.WorkflowInvocationToSubworkflowInvocationAssociation.workflow_step_id == model.WorkflowInvocationStep.table.c.workflow_step_id, - )).scalar_subquery(), - ), - output_dataset_collections=relation( - model.WorkflowInvocationStepOutputDatasetCollectionAssociation, - back_populates='workflow_invocation_step'), - output_datasets=relation(model.WorkflowInvocationStepOutputDatasetAssociation, - back_populates='workflow_invocation_step'), -) - simple_mapping( model.WorkflowInvocationOutputValue, workflow_invocation=relation(model.WorkflowInvocation, backref="output_values"), workflow_invocation_step=relation(model.WorkflowInvocationStep, - foreign_keys=[model.WorkflowInvocationStep.table.c.workflow_invocation_id, model.WorkflowInvocationStep.table.c.workflow_step_id], + foreign_keys=[model.WorkflowInvocationStep.workflow_invocation_id, model.WorkflowInvocationStep.workflow_step_id], primaryjoin=and_( - model.WorkflowInvocationStep.table.c.workflow_invocation_id == model.WorkflowInvocationOutputValue.table.c.workflow_invocation_id, - model.WorkflowInvocationStep.table.c.workflow_step_id == model.WorkflowInvocationOutputValue.table.c.workflow_step_id, + model.WorkflowInvocationStep.workflow_invocation_id == model.WorkflowInvocationOutputValue.table.c.workflow_invocation_id, + model.WorkflowInvocationStep.workflow_step_id == model.WorkflowInvocationOutputValue.table.c.workflow_step_id, ), backref='output_value', viewonly=True @@ -647,6 +618,7 @@ mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( container=relation('JobContainerAssociation', back_populates='job', uselist=False), data_manager_association=relation('DataManagerJobAssociation', back_populates='job', uselist=False), history_dataset_collection_associations=relation('HistoryDatasetCollectionAssociation', back_populates='job'), + workflow_invocation_step=relation('WorkflowInvocationStep', back_populates='job', uselist=False), )) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index ca83b2c9a78..cd4c38150af 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1876,15 +1876,18 @@ class TestImplicitCollectionJobs(BaseTest): cls_, implicit_collection_jobs_job_association, history_dataset_collection_association, + workflow_invocation_step, ): obj = cls_() obj.jobs.append(implicit_collection_jobs_job_association) obj.history_dataset_collection_associations.append(history_dataset_collection_association) + obj.workflow_invocation_step = workflow_invocation_step with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.jobs == [implicit_collection_jobs_job_association] assert stored_obj.history_dataset_collection_associations == [history_dataset_collection_association] + assert stored_obj.workflow_invocation_step.id == workflow_invocation_step.id class TestImplicitCollectionJobsJobAssociation(BaseTest): @@ -4449,8 +4452,8 @@ class TestWorkerProcess(BaseTest): class TestWorkflowInvocationStep(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'workflow_invocation_step' + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_invocation_step' def test_columns( self, From 1d03d051ae56c2ae7eec8d6d08117fe8aac6421f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 10:53:41 -0400 Subject: [PATCH 170/225] Add tests for WorkflowInvocation --- lib/galaxy/model/__init__.py | 8 +-- test/unit/model/test_model.py | 127 ++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 27ddab1dfc7..8a7556d85cf 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6601,10 +6601,10 @@ class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable, RepresentById): non_terminal_states = [states.NEW, states.READY] - def __init__(self): - self.subworkflow_invocations = [] - self.step_states = [] - self.steps = [] + def __init__(self): # TODO no need + self.subworkflow_invocations = [] # TODO instrumented attribute / overwritten + self.step_states = [] # TODO instrumented attribute / overwritten + self.steps = [] # TODO instrumented attribute / overwritten self.workflow_id = None def create_subworkflow_invocation_for_step(self, step): diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index cd4c38150af..24cd9a6125a 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -4450,6 +4450,91 @@ class TestWorkerProcess(BaseTest): assert stored_obj.update_time == update_time +class TestWorkflowInvocation(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'workflow_invocation' + + def test_columns(self, session, cls_, workflow, history): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + state = 'a' + scheduler = 'b' + handler = 'c' + uuid = uuid4() + + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.workflow = workflow + obj.state = state + obj.scheduler = scheduler + obj.handler = handler + obj.uuid = uuid + obj.history = history + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.workflow_id == workflow.id + assert stored_obj.state == state + assert stored_obj.scheduler == scheduler + assert stored_obj.handler == handler + assert stored_obj.uuid == uuid + assert stored_obj.history_id == history.id + + def test_relationships( + self, + session, + cls_, + workflow, + history, + workflow_invocation_output_dataset_association, + workflow_invocation_output_dataset_collection_association, + workflow_invocation_step, + workflow_invocation_to_subworkflow_invocation_association_factory, + workflow_request_input_parameter, + workflow_request_input_step_parameter, + workflow_request_step_state, + workflow_request_to_input_dataset_association, + workflow_request_to_input_dataset_collection_association, + ): + subworkflow_invocation_assoc = workflow_invocation_to_subworkflow_invocation_association_factory() + parent_workflow_invocation_assoc = workflow_invocation_to_subworkflow_invocation_association_factory() + + obj = cls_() + obj.workflow = workflow + obj.history = history + obj.input_parameters.append(workflow_request_input_parameter) + obj.step_states.append(workflow_request_step_state) + obj.input_step_parameters.append(workflow_request_input_step_parameter) + obj.input_datasets.append(workflow_request_to_input_dataset_association) + obj.input_dataset_collections.append(workflow_request_to_input_dataset_collection_association) + obj.subworkflow_invocations.append(subworkflow_invocation_assoc) + obj.steps.append(workflow_invocation_step) + obj.output_dataset_collections.append(workflow_invocation_output_dataset_collection_association) + obj.output_datasets.append(workflow_invocation_output_dataset_association) + obj.parent_workflow_invocation_association.append(parent_workflow_invocation_assoc) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow.id == workflow.id + assert stored_obj.history.id == history.id + + assert stored_obj.input_parameters == [workflow_request_input_parameter] + assert stored_obj.step_states == [workflow_request_step_state] + assert stored_obj.input_step_parameters == [workflow_request_input_step_parameter] + assert stored_obj.input_datasets == [workflow_request_to_input_dataset_association] + assert stored_obj.input_dataset_collections == [workflow_request_to_input_dataset_collection_association] + assert stored_obj.subworkflow_invocations == [subworkflow_invocation_assoc] + assert stored_obj.steps == [workflow_invocation_step] + assert stored_obj.output_dataset_collections == [workflow_invocation_output_dataset_collection_association] + assert stored_obj.output_datasets == [workflow_invocation_output_dataset_association] + assert stored_obj.parent_workflow_invocation_association == [parent_workflow_invocation_assoc] + + class TestWorkflowInvocationStep(BaseTest): def test_table(self, cls_): @@ -5839,6 +5924,18 @@ def workflow_invocation(model, session, workflow): yield from dbcleanup_wrapper(session, wi) +@pytest.fixture +def workflow_invocation_output_dataset_association(model, session): + instance = model.WorkflowInvocationOutputDatasetAssociation() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def workflow_invocation_output_dataset_collection_association(model, session): + instance = model.WorkflowInvocationOutputDatasetCollectionAssociation() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def workflow_invocation_step(model, session, workflow_invocation, workflow_step): wis = model.WorkflowInvocationStep() @@ -5865,6 +5962,36 @@ def workflow_output(model, session, workflow_step): yield from dbcleanup_wrapper(session, wo) +@pytest.fixture +def workflow_request_input_parameter(model, session): + instance = model.WorkflowRequestInputParameter() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def workflow_request_input_step_parameter(model, session): + instance = model.WorkflowRequestInputStepParameter() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def workflow_request_step_state(model, session): + instance = model.WorkflowRequestStepState() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def workflow_request_to_input_dataset_association(model, session): + instance = model.WorkflowRequestToInputDatasetAssociation() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def workflow_request_to_input_dataset_collection_association(model, session): + instance = model.WorkflowRequestToInputDatasetCollectionAssociation() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def workflow_step(model, session, workflow): s = model.WorkflowStep() From b55db9c8941e7c71d20e07f60ed68499e8101093 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 14:17:28 -0400 Subject: [PATCH 171/225] Map WorkflowInvocationStep declaratively --- lib/galaxy/model/__init__.py | 59 +++++++++++++++++++++++++++++++++-- lib/galaxy/model/mapping.py | 42 +------------------------ test/unit/model/test_model.py | 13 ++++++-- 3 files changed, 69 insertions(+), 45 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 8a7556d85cf..a504cde8d95 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2343,6 +2343,7 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, RepresentByI default_permissions = relationship('DefaultHistoryPermissions', back_populates='history') users_shared_with = relationship('HistoryUserShareAssociation', back_populates='history') galaxy_sessions = relationship('GalaxySessionToHistoryAssociation', back_populates='history') + workflow_invocations = relationship('WorkflowInvocation', back_populates='history') update_time = column_property( select(func.max(HistoryAudit.update_time)).where(HistoryAudit.history_id == id).scalar_subquery(), @@ -6588,7 +6589,48 @@ class StoredWorkflowMenuEntry(Base, RepresentById): self.order_index = None -class WorkflowInvocation(UsesCreateAndUpdateTime, Dictifiable, RepresentById): +class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentById): + __tablename__ = 'workflow_invocation' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now, index=True) + workflow_id = Column(Integer, ForeignKey('workflow.id'), index=True, nullable=False) + state = Column(TrimmedString(64), index=True) + scheduler = Column(TrimmedString(255), index=True) + handler = Column(TrimmedString(255), index=True) + uuid = Column(UUIDType()) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + + history = relationship('History', back_populates='workflow_invocations') + input_parameters = relationship('WorkflowRequestInputParameter', + back_populates='workflow_invocation') + step_states = relationship('WorkflowRequestStepState', back_populates='workflow_invocation') + input_step_parameters = relationship('WorkflowRequestInputStepParameter', + back_populates='workflow_invocation') + input_datasets = relationship('WorkflowRequestToInputDatasetAssociation', + back_populates='workflow_invocation') + input_dataset_collections = relationship('WorkflowRequestToInputDatasetCollectionAssociation', + back_populates='workflow_invocation') + subworkflow_invocations = relationship('WorkflowInvocationToSubworkflowInvocationAssociation', + primaryjoin=(lambda: + WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id # type: ignore + == WorkflowInvocation.id), # type: ignore + back_populates='parent_workflow_invocation', + uselist=True, + ) + steps = relationship('WorkflowInvocationStep', back_populates='workflow_invocation') + workflow = relationship('Workflow') + output_dataset_collections = relationship('WorkflowInvocationOutputDatasetCollectionAssociation', + back_populates='workflow_invocation') + output_datasets = relationship('WorkflowInvocationOutputDatasetAssociation', + back_populates='workflow_invocation') + parent_workflow_invocation_association = relationship('WorkflowInvocationToSubworkflowInvocationAssociation', + primaryjoin=(lambda: + WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id # type: ignore + == WorkflowInvocation.id), # type: ignore + back_populates='subworkflow_invocation') + dict_collection_visible_keys = ['id', 'update_time', 'create_time', 'workflow_id', 'history_id', 'uuid', 'state'] dict_element_visible_keys = ['id', 'update_time', 'create_time', 'workflow_id', 'history_id', 'uuid', 'state'] @@ -6957,7 +6999,13 @@ class WorkflowInvocationToSubworkflowInvocationAssociation(Base, Dictifiable, Re uselist=False, ) workflow_step = relationship('WorkflowStep') - + parent_workflow_invocation = relationship('WorkflowInvocation', + primaryjoin=(lambda: + WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id # type: ignore + == WorkflowInvocation.id), # type: ignore + back_populates='subworkflow_invocations', + uselist=False, + ) dict_collection_visible_keys = ['id', 'workflow_step_id', 'workflow_invocation_id', 'subworkflow_invocation_id'] dict_element_visible_keys = ['id', 'workflow_step_id', 'workflow_invocation_id', 'subworkflow_invocation_id'] @@ -6986,6 +7034,7 @@ class WorkflowInvocationStep(Base, Dictifiable, RepresentById): back_populates='workflow_invocation_step') output_datasets = relationship('WorkflowInvocationStepOutputDatasetAssociation', back_populates='workflow_invocation_step') + workflow_invocation = relationship('WorkflowInvocation', back_populates='steps') subworkflow_invocation_id: column_property @@ -7106,6 +7155,7 @@ class WorkflowRequestStepState(Base, Dictifiable, RepresentById): workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) value = Column(MutableJSONType) workflow_step = relationship('WorkflowStep') + workflow_invocation = relationship('WorkflowInvocation', back_populates='step_states') dict_collection_visible_keys = ['id', 'name', 'value', 'workflow_step_id'] @@ -7126,8 +7176,10 @@ class WorkflowRequestToInputDatasetAssociation(Base, Dictifiable, RepresentById) workflow_invocation_id = Column(Integer, ForeignKey("workflow_invocation.id"), index=True) workflow_step_id = Column(Integer, ForeignKey("workflow_step.id")) dataset_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + workflow_step = relationship('WorkflowStep') dataset = relationship('HistoryDatasetAssociation') + workflow_invocation = relationship('WorkflowInvocation', back_populates='input_datasets') history_content_type = "dataset" dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_id', 'name'] @@ -7145,6 +7197,7 @@ class WorkflowRequestToInputDatasetCollectionAssociation(Base, Dictifiable, Repr dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) workflow_step = relationship('WorkflowStep') dataset_collection = relationship('HistoryDatasetCollectionAssociation') + workflow_invocation = relationship('WorkflowInvocation', back_populates='input_dataset_collections') history_content_type = "dataset_collection" dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_collection_id', 'name'] @@ -7159,7 +7212,9 @@ class WorkflowRequestInputStepParameter(Base, Dictifiable, RepresentById): workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id'), index=True) workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) parameter_value = Column(MutableJSONType) + workflow_step = relationship('WorkflowStep') + workflow_invocation = relationship('WorkflowInvocation', back_populates='input_step_parameters') dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'parameter_value'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index affd048a3b5..470a1da87e2 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -28,7 +28,7 @@ from sqlalchemy import ( ) from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.orderinglist import ordering_list -from sqlalchemy.orm import backref, class_mapper, column_property, deferred, object_session, relation +from sqlalchemy.orm import class_mapper, column_property, deferred, object_session, relation from sqlalchemy.orm.collections import attribute_mapped_collection from sqlalchemy.sql import exists @@ -246,18 +246,6 @@ model.Workflow.table = Table( Column("license", TEXT), Column("uuid", UUIDType, nullable=True)) -model.WorkflowInvocation.table = Table( - "workflow_invocation", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now, index=True), - Column("workflow_id", Integer, ForeignKey("workflow.id"), index=True, nullable=False), - Column("state", TrimmedString(64), index=True), - Column("scheduler", TrimmedString(255), index=True), - Column("handler", TrimmedString(255), index=True), - Column('uuid', UUIDType()), - Column("history_id", Integer, ForeignKey("history.id"), index=True)) - model.WorkflowInvocationOutputValue.table = Table( "workflow_invocation_output_value", metadata, Column("id", Integer, primary_key=True), @@ -532,34 +520,6 @@ mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.tabl # returns a list of users that workflow is shared with. model.StoredWorkflow.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore -mapper_registry.map_imperatively(model.WorkflowInvocation, model.WorkflowInvocation.table, properties=dict( - history=relation(model.History, backref=backref('workflow_invocations', uselist=True)), - input_parameters=relation(model.WorkflowRequestInputParameter, back_populates='workflow_invocation'), - step_states=relation(model.WorkflowRequestStepState, backref='workflow_invocation'), - input_step_parameters=relation(model.WorkflowRequestInputStepParameter, - backref='workflow_invocation'), - input_datasets=relation(model.WorkflowRequestToInputDatasetAssociation, - backref='workflow_invocation'), - input_dataset_collections=relation(model.WorkflowRequestToInputDatasetCollectionAssociation, - backref='workflow_invocation'), - subworkflow_invocations=relation(model.WorkflowInvocationToSubworkflowInvocationAssociation, - primaryjoin=(model.WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id == model.WorkflowInvocation.table.c.id), - backref=backref("parent_workflow_invocation", uselist=False), - uselist=True, - ), - steps=relation(model.WorkflowInvocationStep, - backref="workflow_invocation"), - workflow=relation(model.Workflow), - output_dataset_collections=relation(model.WorkflowInvocationOutputDatasetCollectionAssociation, - back_populates='workflow_invocation'), - output_datasets=relation(model.WorkflowInvocationOutputDatasetAssociation, - back_populates='workflow_invocation'), - parent_workflow_invocation_association=relation(model.WorkflowInvocationToSubworkflowInvocationAssociation, - primaryjoin=( - model.WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id == model.WorkflowInvocation.table.c.id), - back_populates='subworkflow_invocation'), -)) - simple_mapping( model.WorkflowInvocationOutputValue, workflow_invocation=relation(model.WorkflowInvocation, backref="output_values"), diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 24cd9a6125a..086837415ac 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1288,6 +1288,7 @@ class TestHistory(BaseTest): default_history_permissions, history_user_share_association, galaxy_session_history_association, + workflow_invocation, ): obj = cls_() obj.user = user @@ -1303,6 +1304,7 @@ class TestHistory(BaseTest): obj.default_permissions.append(default_history_permissions) obj.users_shared_with.append(history_user_share_association) obj.galaxy_sessions.append(galaxy_session_history_association) + obj.workflow_invocations.append(workflow_invocation) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -1322,6 +1324,7 @@ class TestHistory(BaseTest): assert stored_obj.users_shared_with == [history_user_share_association] assert stored_obj.default_permissions == [default_history_permissions] assert stored_obj.galaxy_sessions == [galaxy_session_history_association] + assert stored_obj.workflow_invocations == [workflow_invocation] def test_average_rating(self, session, history, user, history_rating_association_factory): # History has been expunged; to access its deferred properties, @@ -4452,8 +4455,8 @@ class TestWorkerProcess(BaseTest): class TestWorkflowInvocation(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'workflow_invocation' + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_invocation' def test_columns(self, session, cls_, workflow, history): create_time = datetime.now() @@ -4863,15 +4866,21 @@ class TestWorkflowInvocationToSubworkflowInvocationAssociation(BaseTest): workflow_invocation, workflow_step, workflow, + workflow_invocation_factory, ): + parent_workflow_invocation = workflow_invocation_factory() + persist(session, parent_workflow_invocation) + obj = cls_() obj.subworkflow_invocation = workflow_invocation # We need only 1 instance, so we use the fixture obj.workflow_step = workflow_step + obj.parent_workflow_invocation = parent_workflow_invocation with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.subworkflow_invocation.id == workflow_invocation.id assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.parent_workflow_invocation.id == parent_workflow_invocation.id class TestWorkflowOutput(BaseTest): From 1db1675bb423715b0c29e0e5bcad1ee86bfeebd6 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 16:29:29 -0400 Subject: [PATCH 172/225] Map WorkflowInvocationOutputValue declaratively --- lib/galaxy/model/__init__.py | 34 ++++++++++++++++- lib/galaxy/model/mapping.py | 25 ------------ test/unit/model/test_model.py | 71 +++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index a504cde8d95..525d2c5009d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -7035,6 +7035,15 @@ class WorkflowInvocationStep(Base, Dictifiable, RepresentById): output_datasets = relationship('WorkflowInvocationStepOutputDatasetAssociation', back_populates='workflow_invocation_step') workflow_invocation = relationship('WorkflowInvocation', back_populates='steps') + output_value = relationship('WorkflowInvocationOutputValue', + foreign_keys='[WorkflowInvocationStep.workflow_invocation_id, WorkflowInvocationStep.workflow_step_id]', + primaryjoin=(lambda: and_( + WorkflowInvocationStep.workflow_invocation_id == WorkflowInvocationOutputValue.workflow_invocation_id, # type: ignore + WorkflowInvocationStep.workflow_step_id == WorkflowInvocationOutputValue.workflow_step_id, # type: ignore + )), + back_populates='workflow_invocation_step', + viewonly=True + ) subworkflow_invocation_id: column_property @@ -7257,8 +7266,31 @@ class WorkflowInvocationOutputDatasetCollectionAssociation(Base, Dictifiable, Re dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_collection_id', 'name'] -class WorkflowInvocationOutputValue(Dictifiable, RepresentById): +class WorkflowInvocationOutputValue(Base, Dictifiable, RepresentById): """Represents a link to a specified or computed workflow parameter.""" + __tablename__ = 'workflow_invocation_output_value' + + id = Column(Integer, primary_key=True) + workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id'), index=True) + workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) + workflow_output_id = Column(Integer, ForeignKey('workflow_output.id'), index=True) + value = Column(MutableJSONType) + + workflow_invocation = relationship('WorkflowInvocation', backref="output_values") + + workflow_invocation_step = relationship('WorkflowInvocationStep', + foreign_keys='[WorkflowInvocationStep.workflow_invocation_id, WorkflowInvocationStep.workflow_step_id]', + primaryjoin=(lambda: and_( + WorkflowInvocationStep.workflow_invocation_id == WorkflowInvocationOutputValue.workflow_invocation_id, # type: ignore + WorkflowInvocationStep.workflow_step_id == WorkflowInvocationOutputValue.workflow_step_id, # type: ignore + )), + back_populates='output_value', + viewonly=True + ) + + workflow_step = relationship('WorkflowStep') + workflow_output = relationship('WorkflowOutput') + dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'value'] diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 470a1da87e2..eadcb11611a 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -246,15 +246,6 @@ model.Workflow.table = Table( Column("license", TEXT), Column("uuid", UUIDType, nullable=True)) -model.WorkflowInvocationOutputValue.table = Table( - "workflow_invocation_output_value", metadata, - Column("id", Integer, primary_key=True), - Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id"), index=True), - Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")), - Column("workflow_output_id", Integer, ForeignKey("workflow_output.id"), index=True), - Column("value", MutableJSONType), -) - # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -520,22 +511,6 @@ mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.tabl # returns a list of users that workflow is shared with. model.StoredWorkflow.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore -simple_mapping( - model.WorkflowInvocationOutputValue, - workflow_invocation=relation(model.WorkflowInvocation, backref="output_values"), - workflow_invocation_step=relation(model.WorkflowInvocationStep, - foreign_keys=[model.WorkflowInvocationStep.workflow_invocation_id, model.WorkflowInvocationStep.workflow_step_id], - primaryjoin=and_( - model.WorkflowInvocationStep.workflow_invocation_id == model.WorkflowInvocationOutputValue.table.c.workflow_invocation_id, - model.WorkflowInvocationStep.workflow_step_id == model.WorkflowInvocationOutputValue.table.c.workflow_step_id, - ), - backref='output_value', - viewonly=True - ), - workflow_step=relation(model.WorkflowStep), - workflow_output=relation(model.WorkflowOutput), -) - # Set up proxy so that # Page.users_shared_with # returns a list of users that page is shared with. diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 086837415ac..d5e69084f18 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -4591,10 +4591,17 @@ class TestWorkflowInvocationStep(BaseTest): implicit_collection_jobs, workflow_invocation_step_output_dataset_collection_association, workflow_invocation_step_output_dataset_association, + workflow_invocation_output_value, ): session.add(job) # needed for a lazy load of an attr session.add(implicit_collection_jobs) # needed for a lazy load of an attr + # setup workflow_invocation_output_value to test the output_value attribute + output_value = workflow_invocation_output_value + output_value.workflow_invocation = workflow_invocation + output_value.workflow_step = workflow_step + persist(session, output_value) + obj = cls_() obj.workflow_invocation = workflow_invocation obj.workflow_step = workflow_step @@ -4611,6 +4618,7 @@ class TestWorkflowInvocationStep(BaseTest): assert stored_obj.implicit_collection_jobs.id == implicit_collection_jobs.id assert stored_obj.output_dataset_collections == [workflow_invocation_step_output_dataset_collection_association] assert stored_obj.output_datasets == [workflow_invocation_step_output_dataset_association] + assert stored_obj.output_value.id == workflow_invocation_output_value.id def test_subworkflow_invocation_attribute( self, @@ -4752,6 +4760,63 @@ class TestWorkflowInvocationOutputDatasetCollectionAssociation(BaseTest): assert stored_obj.workflow_output.id == workflow_output.id +class TestWorkflowInvocationOutputValue(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_invocation_output_value' + + def test_columns( + self, + session, + cls_, + workflow_invocation, + workflow_step, + workflow_output, + ): + value = 'a' + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.workflow_output = workflow_output + obj.value = value + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.workflow_output_id == workflow_output.id + assert stored_obj.value == value + + def test_relationships( + self, + session, + cls_, + workflow_invocation, + workflow_step, + workflow_output, + workflow_invocation_step, + ): + # setup workflow_invocation_step to test the workflow_invocation_step attribute + workflow_invocation_step.workflow_invocation = workflow_invocation + workflow_invocation_step.workflow_step = workflow_step + persist(session, workflow_invocation_step) + + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.workflow_output = workflow_output + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.workflow_output.id == workflow_output.id + assert stored_obj.workflow_invocation_step[0].id == workflow_invocation_step.id + + delete_from_database(session, [workflow_invocation_step]) + + class TestWorkflowInvocationStepOutputDatasetAssociation(BaseTest): def test_table(self, cls_): @@ -5945,6 +6010,12 @@ def workflow_invocation_output_dataset_collection_association(model, session): yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def workflow_invocation_output_value(model, session): + instance = model.WorkflowInvocationOutputValue() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def workflow_invocation_step(model, session, workflow_invocation, workflow_step): wis = model.WorkflowInvocationStep() From 409998ff03e3501a12fc1a6f3e173f25a7b6e161 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 17:47:48 -0400 Subject: [PATCH 173/225] Add unit tests for Workflow --- test/unit/model/test_model.py | 80 +++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index d5e69084f18..30345064bb2 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -4453,6 +4453,70 @@ class TestWorkerProcess(BaseTest): assert stored_obj.update_time == update_time +class TestWorkflow(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'workflow' + + def test_columns(self, session, cls_, stored_workflow, workflow): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + name = 'a' + has_cycles = True + has_errors = True + reports_config = 'b' + creator_metadata = 'c' + license = 'd' + uuid = uuid4() + + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.stored_workflow = stored_workflow + obj.parent_workflow_id = workflow.id + obj.name = name + obj.has_cycles = has_cycles + obj.has_errors = has_errors + obj.reports_config = reports_config + obj.creator_metadata = creator_metadata + obj.license = license + obj.uuid = uuid + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id, unique=True) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.stored_workflow_id == stored_workflow.id + assert stored_obj.parent_workflow_id == workflow.id + assert stored_obj.name == name + assert stored_obj.has_cycles == has_cycles + assert stored_obj.has_errors == has_errors + assert stored_obj.reports_config == reports_config + assert stored_obj.creator_metadata == creator_metadata + assert stored_obj.license == license + assert stored_obj.uuid == uuid + + def test_relationships(self, session, cls_, stored_workflow, workflow, workflow_step_factory): + obj = cls_() + obj.stored_workflow = stored_workflow + obj.parent_workflow_id = workflow.id + + # Setup workflow steps to test attribures: steps, parent_workflow_step + workflow_step = workflow_step_factory(workflow=obj) + parent_workflow_step = workflow_step_factory(subworkflow=obj) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id, unique=True) + assert stored_obj.stored_workflow_id == stored_workflow.id + assert stored_obj.parent_workflow_id == workflow.id + assert stored_obj.steps[0].id == workflow_step.id + assert stored_obj.step_count == 1 + assert stored_obj.parent_workflow_steps[0].id == parent_workflow_step.id + + delete_from_database(session, [workflow_step, parent_workflow_step]) + + class TestWorkflowInvocation(BaseTest): def test_table(self, cls_): @@ -6202,9 +6266,12 @@ def workflow_step_connection_factory(model): @pytest.fixture -def workflow_step_factory(model): +def workflow_step_factory(model, workflow): def make_instance(*args, **kwds): - return model.WorkflowStep(*args, **kwds) + instance = model.WorkflowStep() + instance.workflow = kwds.get('workflow', workflow) + instance.subworkflow = kwds.get('subworkflow') + return instance return make_instance @@ -6260,13 +6327,18 @@ def delete_from_database(session, objects): session.execute(stmt) -def get_stored_obj(session, cls, obj_id=None, where_clause=None): +def get_stored_obj(session, cls, obj_id=None, where_clause=None, unique=False): # Either obj_id or where_clause must be provided, but not both assert bool(obj_id) ^ (where_clause is not None) if where_clause is None: where_clause = get_default_where_clause(cls, obj_id) stmt = select(cls).where(where_clause) - return session.execute(stmt).scalar_one() + result = session.execute(stmt) + # unique() is required if result contains joint eager loads against collections + # https://gerrit.sqlalchemy.org/c/sqlalchemy/sqlalchemy/+/2253 + if unique: + result = result.unique() + return result.scalar_one() def get_default_where_clause(cls, obj_id): From 9ec6ed889f22fcf137cae86198759793dc8762c3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 17:58:17 -0400 Subject: [PATCH 174/225] Map Workflow declaratively --- lib/galaxy/model/__init__.py | 36 ++++++++++++++++++++++++++++++++- lib/galaxy/model/mapping.py | 38 ++--------------------------------- test/unit/model/test_model.py | 4 ++-- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 525d2c5009d..7912342a69c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6047,13 +6047,42 @@ class StoredWorkflow(HasTags, Dictifiable, RepresentById): return rval -class Workflow(Dictifiable, RepresentById): +class Workflow(Base, Dictifiable, RepresentById): """ Workflow represents a revision of a :class:`galaxy.model.StoredWorkflow`. A new instance is created for each workflow revision and provides a common parent for the workflow steps. See :class:`galaxy.model.WorkflowStep` for more information """ + __tablename__ = 'workflow' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + # workflows will belong to either a stored workflow or a parent/nesting workflow. + stored_workflow_id = Column(Integer, ForeignKey('stored_workflow.id'), index=True, nullable=True) + parent_workflow_id = Column(Integer, ForeignKey('workflow.id'), index=True, nullable=True) + name = Column(TEXT) + has_cycles = Column(Boolean) + has_errors = Column(Boolean) + reports_config = Column(MutableJSONType) + creator_metadata = Column(MutableJSONType) + license = Column(TEXT) + uuid = Column(UUIDType, nullable=True) + + steps = relationship('WorkflowStep', + backref='workflow', + primaryjoin=(lambda: Workflow.id == WorkflowStep.workflow_id), # type: ignore + order_by=lambda: asc(WorkflowStep.order_index), # type: ignore + cascade="all, delete-orphan", + lazy=False) + parent_workflow_steps = relationship( + 'WorkflowStep', + primaryjoin=(lambda: Workflow.id == WorkflowStep.subworkflow_id), # type: ignore + back_populates='subworkflow') + + step_count: column_property + dict_collection_visible_keys = ['name', 'has_cycles', 'has_errors'] dict_element_visible_keys = ['name', 'has_cycles', 'has_errors'] input_step_types = ['data_input', 'data_collection_input', 'parameter_input'] @@ -8790,3 +8819,8 @@ WorkflowInvocationStep.subworkflow_invocation_id = column_property( WorkflowInvocationToSubworkflowInvocationAssociation.workflow_step_id == WorkflowInvocationStep.workflow_step_id, )).scalar_subquery(), ) + +Workflow.step_count = column_property( + select(func.count(WorkflowStep.id)).where(Workflow.id == WorkflowStep.workflow_id).scalar_subquery(), + deferred=True +) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index eadcb11611a..dba462a22dc 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -9,7 +9,6 @@ from typing import Optional, Type from sqlalchemy import ( and_, - asc, Boolean, Column, DateTime, @@ -230,22 +229,6 @@ model.StoredWorkflow.table = Table( Index('ix_stored_workflow_slug', 'slug', mysql_length=200), ) -model.Workflow.table = Table( - "workflow", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - # workflows will belong to either a stored workflow or a parent/nesting workflow. - Column("stored_workflow_id", Integer, ForeignKey("stored_workflow.id"), index=True, nullable=True), - Column("parent_workflow_id", Integer, ForeignKey("workflow.id"), index=True, nullable=True), - Column("name", TEXT), - Column("has_cycles", Boolean), - Column("has_errors", Boolean), - Column("reports_config", MutableJSONType), - Column("creator_metadata", MutableJSONType), - Column("license", TEXT), - Column("uuid", UUIDType, nullable=True)) - # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -453,23 +436,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod # ), # ) -mapper_registry.map_imperatively(model.Workflow, model.Workflow.table, properties=dict( - steps=relation(model.WorkflowStep, - backref='workflow', - primaryjoin=(model.Workflow.table.c.id == model.WorkflowStep.workflow_id), - order_by=asc(model.WorkflowStep.order_index), - cascade="all, delete-orphan", - lazy=False), - step_count=column_property( - select(func.count(model.WorkflowStep.id)).where(model.Workflow.table.c.id == model.WorkflowStep.workflow_id).scalar_subquery(), - deferred=True - ), - parent_workflow_steps=relation( - model.WorkflowStep, - primaryjoin=(lambda: model.Workflow.id == model.WorkflowStep.subworkflow_id), # type: ignore - back_populates='subworkflow'), -)) - mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.table, properties=dict( user=relation(model.User, primaryjoin=(model.User.table.c.id == model.StoredWorkflow.table.c.user_id), @@ -477,11 +443,11 @@ mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.tabl workflows=relation(model.Workflow, backref='stored_workflow', cascade="all, delete-orphan", - primaryjoin=(model.StoredWorkflow.table.c.id == model.Workflow.table.c.stored_workflow_id), + primaryjoin=(model.StoredWorkflow.table.c.id == model.Workflow.stored_workflow_id), order_by=-model.Workflow.id), latest_workflow=relation(model.Workflow, post_update=True, - primaryjoin=(model.StoredWorkflow.table.c.latest_workflow_id == model.Workflow.table.c.id), + primaryjoin=(model.StoredWorkflow.table.c.latest_workflow_id == model.Workflow.id), lazy=False), tags=relation(model.StoredWorkflowTagAssociation, order_by=model.StoredWorkflowTagAssociation.id, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 30345064bb2..7ac229d4951 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -4455,8 +4455,8 @@ class TestWorkerProcess(BaseTest): class TestWorkflow(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'workflow' + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow' def test_columns(self, session, cls_, stored_workflow, workflow): create_time = datetime.now() From 90220b98f1e784347ae39da8b9634f8afa6c6454 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 19:05:12 -0400 Subject: [PATCH 175/225] Add unit test for StoredWorkflow --- test/unit/model/test_model.py | 141 ++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 7ac229d4951..40faf8a691c 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3601,6 +3601,115 @@ class TestRole(BaseTest): assert stored_obj.users == [user_role_association] +class TestStoredWorkflow(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'stored_workflow' + # assert has_index(cls_.__table__, ('ix_stored_workflow_slug', 'slug',)) + + def test_columns(self, session, cls_, workflow, user): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + name = 'a' + deleted = True + hidden = True + importable = True + slug = 'b' + from_path = 'c' + published = True + + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.user = user + obj.latest_workflow = workflow + obj.name = name + obj.deleted = deleted + obj.hidden = hidden + obj.importable = importable + obj.slug = slug + obj.from_path = from_path + obj.published = published + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id, unique=True) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.user_id == user.id + assert stored_obj.latest_workflow_id == workflow.id + assert stored_obj.name == name + assert stored_obj.deleted == deleted + assert stored_obj.hidden == hidden + assert stored_obj.importable == importable + assert stored_obj.slug == slug + assert stored_obj.from_path == from_path + assert stored_obj.published == published + + def test_relationships( + self, + session, + cls_, + workflow, + user, + workflow_factory, + stored_workflow_tag_association, + stored_workflow_annotation_association, + stored_workflow_rating_association, + stored_workflow_tag_association_factory, + stored_workflow_user_share_association, + ): + obj = cls_() + obj.user = user + obj.latest_workflow = workflow + obj.tags.append(stored_workflow_tag_association) + obj.annotations.append(stored_workflow_annotation_association) + obj.ratings.append(stored_workflow_rating_association) + obj.users_shared_with.append(stored_workflow_user_share_association) + + # setup workflow for testing workflows attribure + wf = workflow_factory() + wf.stored_workflow = obj + + # setup owner tag association for testing owner_tags + tag_assoc2 = stored_workflow_tag_association_factory() + tag_assoc2.user = user + tag_assoc2.stored_workflow = obj + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id, unique=True) + assert stored_obj.user.id == user.id + assert stored_obj.latest_workflow.id == workflow.id + assert stored_obj.workflows[0].id == wf.id + assert stored_obj.annotations == [stored_workflow_annotation_association] + assert stored_obj.ratings == [stored_workflow_rating_association] + # This doesn't test the average amount, just the mapping. + assert stored_obj.average_rating == stored_workflow_rating_association.rating + + assert len(stored_obj.tags) == 2 + assert stored_workflow_tag_association in stored_obj.tags + assert tag_assoc2 in stored_obj.tags + assert stored_obj.owner_tags == [tag_assoc2] + assert stored_obj.users_shared_with == [stored_workflow_user_share_association] + + delete_from_database(session, [wf]) + + def test_average_rating(self, session, stored_workflow, user, stored_workflow_rating_association_factory): + # StoredWorkflow has been expunged; to access its deferred properties, + # it needs to be added back to the session. + session.add(stored_workflow) + assert stored_workflow.average_rating is None # With no ratings, we expect None. + # Create ratings + to_cleanup = [] + for rating in (1, 2, 3, 4, 5): + sw_rating_assoc = stored_workflow_rating_association_factory(user, stored_workflow, rating) + persist(session, sw_rating_assoc) + to_cleanup.append(sw_rating_assoc) + assert stored_workflow.average_rating == 3.0 # Expect average after ratings added. + # Cleanup: remove ratings from database + delete_from_database(session, to_cleanup) + + class TestStoredWorkflowAnnotationAssociation(BaseTest): def test_table(self, cls_): @@ -5952,12 +6061,30 @@ def stored_workflow(model, session, user): yield from dbcleanup_wrapper(session, w) +@pytest.fixture +def stored_workflow_annotation_association(model, session): + haa = model.StoredWorkflowAnnotationAssociation() + yield from dbcleanup_wrapper(session, haa) + + +@pytest.fixture +def stored_workflow_rating_association(model, session): + hda = model.StoredWorkflowRatingAssociation(None, None) + yield from dbcleanup_wrapper(session, hda) + + @pytest.fixture def stored_workflow_tag_association(model, session): swta = model.StoredWorkflowTagAssociation() yield from dbcleanup_wrapper(session, swta) +@pytest.fixture +def stored_workflow_user_share_association(model, session): + husa = model.StoredWorkflowUserShareAssociation() + yield from dbcleanup_wrapper(session, husa) + + @pytest.fixture def tag(model, session): t = model.Tag() @@ -6219,6 +6346,20 @@ def page_rating_association_factory(model): return make_instance +@pytest.fixture +def stored_workflow_rating_association_factory(model): + def make_instance(*args, **kwds): + return model.StoredWorkflowRatingAssociation(*args, **kwds) + return make_instance + + +@pytest.fixture +def stored_workflow_tag_association_factory(model): + def make_instance(*args, **kwds): + return model.StoredWorkflowTagAssociation(*args, **kwds) + return make_instance + + @pytest.fixture def visualization_rating_association_factory(model): def make_instance(*args, **kwds): From 70ffeaccf9ea97cb947c9928355c4e4dbe92d160 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 20:20:40 -0400 Subject: [PATCH 176/225] Map StoredWorkflow declaratively --- lib/galaxy/model/__init__.py | 60 ++++++++++++++++++++++++++++++++- lib/galaxy/model/mapping.py | 62 +++-------------------------------- test/unit/model/test_model.py | 6 ++-- 3 files changed, 66 insertions(+), 62 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 7912342a69c..4149007255b 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5995,7 +5995,7 @@ class UCI: self.user = None -class StoredWorkflow(HasTags, Dictifiable, RepresentById): +class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): """ StoredWorkflow represents the root node of a tree of objects that compose a workflow, including workflow revisions, steps, and subworkflows. It is responsible for the metadata associated with a workflow including owner, name, published, and create/update time. @@ -6003,6 +6003,56 @@ class StoredWorkflow(HasTags, Dictifiable, RepresentById): Each time a workflow is modified a revision is created, represented by a new :class:`galaxy.model.Workflow` instance. See :class:`galaxy.model.Workflow` for more information """ + __tablename__ = 'stored_workflow' + __table_args__ = ( + Index('ix_stored_workflow_slug', 'slug', mysql_length=200), + ) + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now, index=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=False) + latest_workflow_id = Column(Integer, + ForeignKey('workflow.id', use_alter=True, name='stored_workflow_latest_workflow_id_fk'), index=True) + name = Column(TEXT) + deleted = Column(Boolean, default=False) + hidden = Column(Boolean, default=False) + importable = Column(Boolean, default=False) + slug = Column(TEXT) + from_path = Column(TEXT) + published = Column(Boolean, index=True, default=False) + + user = relationship('User', + primaryjoin=(lambda: User.table.c.id == StoredWorkflow.user_id), # type: ignore + back_populates='stored_workflows') + workflows = relationship('Workflow', + back_populates='stored_workflow', + cascade="all, delete-orphan", + primaryjoin=(lambda: StoredWorkflow.id == Workflow.stored_workflow_id), # type: ignore + order_by='-Workflow.id') + latest_workflow = relationship('Workflow', + post_update=True, + primaryjoin=(lambda: StoredWorkflow.latest_workflow_id == Workflow.id), # type: ignore + lazy=False) + tags = relationship('StoredWorkflowTagAssociation', + order_by='StoredWorkflowTagAssociation.id', + back_populates="stored_workflow") + owner_tags = relationship('StoredWorkflowTagAssociation', + primaryjoin=(lambda: + and_(StoredWorkflow.id == StoredWorkflowTagAssociation.stored_workflow_id, # type: ignore + StoredWorkflow.user_id == StoredWorkflowTagAssociation.user_id) # type: ignore + ), + viewonly=True, + order_by='StoredWorkflowTagAssociation.id') + annotations = relationship('StoredWorkflowAnnotationAssociation', + order_by='StoredWorkflowAnnotationAssociation.id', + back_populates="stored_workflow") + ratings = relationship('StoredWorkflowRatingAssociation', + order_by='StoredWorkflowRatingAssociation.id', + back_populates="stored_workflow") + users_shared_with = relationship('StoredWorkflowUserShareAssociation', back_populates='stored_workflow') + + average_rating: column_property dict_collection_visible_keys = ['id', 'name', 'create_time', 'update_time', 'published', 'deleted', 'hidden'] dict_element_visible_keys = ['id', 'name', 'create_time', 'update_time', 'published', 'deleted', 'hidden'] @@ -6080,6 +6130,9 @@ class Workflow(Base, Dictifiable, RepresentById): 'WorkflowStep', primaryjoin=(lambda: Workflow.id == WorkflowStep.subworkflow_id), # type: ignore back_populates='subworkflow') + stored_workflow = relationship('StoredWorkflow', + primaryjoin=(lambda: StoredWorkflow.id == Workflow.stored_workflow_id), # type: ignore + back_populates='workflows') step_count: column_property @@ -8824,3 +8877,8 @@ Workflow.step_count = column_property( select(func.count(WorkflowStep.id)).where(Workflow.id == WorkflowStep.workflow_id).scalar_subquery(), deferred=True ) + +StoredWorkflow.average_rating = column_property( + select(func.avg(StoredWorkflowRatingAssociation.rating)).where(StoredWorkflowRatingAssociation.stored_workflow_id == StoredWorkflow.id).scalar_subquery(), + deferred=True +) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index dba462a22dc..da53bf09abb 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -15,8 +15,6 @@ from sqlalchemy import ( desc, false, ForeignKey, - func, - Index, Integer, not_, Numeric, @@ -211,24 +209,6 @@ model.Job.table = Table( Column("params", TrimmedString(255), index=True), Column("handler", TrimmedString(255), index=True)) -model.StoredWorkflow.table = Table( - "stored_workflow", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now, index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True, nullable=False), - Column("latest_workflow_id", Integer, - ForeignKey("workflow.id", use_alter=True, name='stored_workflow_latest_workflow_id_fk'), index=True), - Column("name", TEXT), - Column("deleted", Boolean, default=False), - Column("hidden", Boolean, default=False), - Column("importable", Boolean, default=False), - Column("slug", TEXT), - Column("from_path", TEXT), - Column("published", Boolean, index=True, default=False), - Index('ix_stored_workflow_slug', 'slug', mysql_length=200), -) - # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -348,8 +328,8 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( stored_workflow_menu_entries=relation(model.StoredWorkflowMenuEntry, primaryjoin=( (model.StoredWorkflowMenuEntry.user_id == model.User.table.c.id) - & (model.StoredWorkflowMenuEntry.stored_workflow_id == model.StoredWorkflow.table.c.id) - & not_(model.StoredWorkflow.table.c.deleted) + & (model.StoredWorkflowMenuEntry.stored_workflow_id == model.StoredWorkflow.id) + & not_(model.StoredWorkflow.deleted) ), backref="user", cascade="all, delete-orphan", @@ -369,6 +349,8 @@ mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( data_manager_histories=relation(model.DataManagerHistoryAssociation, back_populates='user'), workflows_shared_by_others=relation(model.StoredWorkflowUserShareAssociation, back_populates='user'), roles=relation(model.UserRoleAssociation, back_populates='user'), + stored_workflows=relation(model.StoredWorkflow, back_populates='user', + primaryjoin=(lambda: model.User.table.c.id == model.StoredWorkflow.user_id)), # type: ignore )) # Set up proxy so that this syntax is possible: @@ -436,42 +418,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, mod # ), # ) -mapper_registry.map_imperatively(model.StoredWorkflow, model.StoredWorkflow.table, properties=dict( - user=relation(model.User, - primaryjoin=(model.User.table.c.id == model.StoredWorkflow.table.c.user_id), - backref='stored_workflows'), - workflows=relation(model.Workflow, - backref='stored_workflow', - cascade="all, delete-orphan", - primaryjoin=(model.StoredWorkflow.table.c.id == model.Workflow.stored_workflow_id), - order_by=-model.Workflow.id), - latest_workflow=relation(model.Workflow, - post_update=True, - primaryjoin=(model.StoredWorkflow.table.c.latest_workflow_id == model.Workflow.id), - lazy=False), - tags=relation(model.StoredWorkflowTagAssociation, - order_by=model.StoredWorkflowTagAssociation.id, - back_populates="stored_workflow"), - owner_tags=relation(model.StoredWorkflowTagAssociation, - primaryjoin=( - and_(model.StoredWorkflow.table.c.id == model.StoredWorkflowTagAssociation.stored_workflow_id, - model.StoredWorkflow.table.c.user_id == model.StoredWorkflowTagAssociation.user_id) - ), - viewonly=True, - order_by=model.StoredWorkflowTagAssociation.id), - annotations=relation(model.StoredWorkflowAnnotationAssociation, - order_by=model.StoredWorkflowAnnotationAssociation.id, - back_populates="stored_workflow"), - ratings=relation(model.StoredWorkflowRatingAssociation, - order_by=model.StoredWorkflowRatingAssociation.id, - back_populates="stored_workflow"), - average_rating=column_property( - select(func.avg(model.StoredWorkflowRatingAssociation.rating)).where(model.StoredWorkflowRatingAssociation.stored_workflow_id == model.StoredWorkflow.table.c.id).scalar_subquery(), - deferred=True - ), - users_shared_with=relation(model.StoredWorkflowUserShareAssociation, back_populates='stored_workflow'), -)) - # Set up proxy so that # StoredWorkflow.users_shared_with # returns a list of users that workflow is shared with. diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 40faf8a691c..654d55e7e51 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -3603,9 +3603,9 @@ class TestRole(BaseTest): class TestStoredWorkflow(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'stored_workflow' - # assert has_index(cls_.__table__, ('ix_stored_workflow_slug', 'slug',)) + def test_table(self, cls_): + assert cls_.__tablename__ == 'stored_workflow' + assert has_index(cls_.__table__, ('slug',)) def test_columns(self, session, cls_, workflow, user): create_time = datetime.now() From 74878055ae0327aeb39e9fe7a6847cbf75df3aac Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 22:10:59 -0400 Subject: [PATCH 177/225] Map ImplicitlyConvertedDatasetAssociation declaratively --- lib/galaxy/model/__init__.py | 27 ++++++++++++++++- lib/galaxy/model/mapping.py | 42 ++++++++------------------ test/unit/model/test_model.py | 57 +++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 30 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4149007255b..e0ad31eeb3c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4970,7 +4970,32 @@ class LibraryDatasetDatasetInfoAssociation(RepresentById): return True # always allow inheriting, used for replacement -class ImplicitlyConvertedDatasetAssociation(RepresentById): +class ImplicitlyConvertedDatasetAssociation(Base, RepresentById): + __tablename__ = 'implicitly_converted_dataset_association' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + hda_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True, nullable=True) + ldda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) + hda_parent_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + ldda_parent_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) + deleted = Column(Boolean, index=True, default=False) + metadata_safe = Column(Boolean, index=True, default=True) + type = Column(TrimmedString(255)) + + parent_hda = relationship('HistoryDatasetAssociation', + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_parent_id # type: ignore + == HistoryDatasetAssociation.id), # type: ignore + back_populates='implicitly_converted_datasets') + dataset_ldda = relationship('LibraryDatasetDatasetAssociation', + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_id # type: ignore + == LibraryDatasetDatasetAssociation.id), # type: ignore + back_populates='implicitly_converted_parent_datasets') + dataset = relationship('HistoryDatasetAssociation', + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_id # type: ignore + == HistoryDatasetAssociation.id), # type: ignore + back_populates='implicitly_converted_parent_datasets') def __init__(self, id=None, parent=None, dataset=None, file_type=None, deleted=False, purged=False, metadata_safe=True): self.id = id diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index da53bf09abb..6f681461999 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -118,19 +118,6 @@ model.Dataset.table = Table( Column('total_size', Numeric(15, 0)), Column('uuid', UUIDType())) -model.ImplicitlyConvertedDatasetAssociation.table = Table( - "implicitly_converted_dataset_association", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("hda_id", Integer, ForeignKey("history_dataset_association.id"), index=True, nullable=True), - Column("ldda_id", Integer, ForeignKey("library_dataset_dataset_association.id"), index=True, nullable=True), - Column("hda_parent_id", Integer, ForeignKey("history_dataset_association.id"), index=True), - Column("ldda_parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), index=True), - Column("deleted", Boolean, index=True, default=False), - Column("metadata_safe", Boolean, index=True, default=True), - Column("type", TrimmedString(255))) - model.LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_association", metadata, Column("id", Integer, primary_key=True), @@ -254,6 +241,14 @@ simple_mapping(model.HistoryDatasetAssociation, creating_job_associations=relation( model.JobToOutputDatasetAssociation, back_populates='dataset'), history=relation(model.History, back_populates='datasets'), + implicitly_converted_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, + primaryjoin=(lambda: model.ImplicitlyConvertedDatasetAssociation.hda_parent_id # type: ignore + == model.HistoryDatasetAssociation.id), # type: ignore + back_populates='parent_hda'), + implicitly_converted_parent_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, + primaryjoin=(lambda: model.ImplicitlyConvertedDatasetAssociation.hda_id # type: ignore + == model.HistoryDatasetAssociation.id), # type: ignore + back_populates='dataset') ) simple_mapping(model.Dataset, @@ -281,21 +276,6 @@ simple_mapping(model.Dataset, genome_index_tool_data=relation(model.GenomeIndexToolData, back_populates='dataset'), ) -mapper_registry.map_imperatively(model.ImplicitlyConvertedDatasetAssociation, model.ImplicitlyConvertedDatasetAssociation.table, properties=dict( - parent_hda=relation(model.HistoryDatasetAssociation, - primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.hda_parent_id - == model.HistoryDatasetAssociation.table.c.id), - backref='implicitly_converted_datasets'), - dataset_ldda=relation(model.LibraryDatasetDatasetAssociation, - primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.ldda_id - == model.LibraryDatasetDatasetAssociation.table.c.id), - backref="implicitly_converted_parent_datasets"), - dataset=relation(model.HistoryDatasetAssociation, - primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.hda_id - == model.HistoryDatasetAssociation.table.c.id), - backref="implicitly_converted_parent_datasets") -)) - # Set up proxy so that # History.users_shared_with # returns a list of users that history is shared with. @@ -376,7 +356,7 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.L == model.HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id), backref='copied_from_library_dataset_dataset_association'), implicitly_converted_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, - primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.ldda_parent_id + primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.ldda_parent_id == model.LibraryDatasetDatasetAssociation.table.c.id), backref='parent_ldda'), tags=relation(model.LibraryDatasetDatasetAssociationTagAssociation, @@ -393,6 +373,10 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.L model.JobToInputLibraryDatasetAssociation, back_populates='dataset'), creating_job_associations=relation( model.JobToOutputLibraryDatasetAssociation, back_populates='dataset'), + implicitly_converted_parent_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, + primaryjoin=(lambda: model.ImplicitlyConvertedDatasetAssociation.ldda_id # type: ignore + == model.LibraryDatasetDatasetAssociation.id), # type: ignore + back_populates='dataset_ldda'), )) mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, model.LibraryDatasetDatasetInfoAssociation.table, properties=dict( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 654d55e7e51..c44950f8ea9 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1926,6 +1926,63 @@ class TestImplicitCollectionJobsJobAssociation(BaseTest): assert stored_obj.job.id == job.id +class TestImplicitlyConvertedDatasetAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'implicitly_converted_dataset_association' + + def test_columns( + self, + session, + cls_, + history_dataset_association, + library_dataset_dataset_association, + ): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + deleted = True + metadata_safe = False + type = 'a' + + # Test this combination (dataset/parent, hda/ldda); assume this is sufficient for other combinations. + obj = cls_(parent=library_dataset_dataset_association, dataset=history_dataset_association) + obj.create_time = create_time + obj.update_time = update_time + obj.deleted = deleted + obj.metadata_safe = metadata_safe + obj.type = type + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.hda_id == history_dataset_association.id + assert stored_obj.ldda_id is None + assert stored_obj.hda_parent_id is None + assert stored_obj.ldda_parent_id == library_dataset_dataset_association.id + assert stored_obj.deleted == deleted + assert stored_obj.metadata_safe == metadata_safe + assert stored_obj.type == type + + def test_relationships( + self, + session, + cls_, + history_dataset_association, + library_dataset_dataset_association, + ): + # Switch hda and ldda (different from test_columns() setup) + obj = cls_(parent=history_dataset_association, dataset=library_dataset_dataset_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset is None + assert stored_obj.dataset_ldda.id == library_dataset_dataset_association.id + assert stored_obj.parent_hda.id == history_dataset_association.id + assert stored_obj.parent_ldda is None + + class TestImplicitlyCreatedDatasetCollectionInput(BaseTest): def test_table(self, cls_): From aef4646192b135aa534b8238f93434b3ac7840b8 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 11 Aug 2021 22:33:23 -0400 Subject: [PATCH 178/225] Map LibraryDatasetDatasetInfoAssoc declaratively --- lib/galaxy/model/__init__.py | 26 +++++++++++++++++++++- lib/galaxy/model/mapping.py | 23 ------------------- test/unit/model/test_model.py | 42 +++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e0ad31eeb3c..4804009cf8d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4958,7 +4958,31 @@ class LibraryFolderInfoAssociation(Base, RepresentById): self.inheritable = inheritable -class LibraryDatasetDatasetInfoAssociation(RepresentById): +class LibraryDatasetDatasetInfoAssociation(Base, RepresentById): + __tablename__ = 'library_dataset_dataset_info_association' + + id = Column(Integer, primary_key=True) + library_dataset_dataset_association_id = Column(Integer, + ForeignKey('library_dataset_dataset_association.id'), nullable=True, index=True) + form_definition_id = Column(Integer, ForeignKey('form_definition.id'), index=True) + form_values_id = Column(Integer, ForeignKey('form_values.id'), index=True) + deleted = Column(Boolean, index=True, default=False) + + library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', + primaryjoin=(lambda: + (LibraryDatasetDatasetInfoAssociation.library_dataset_dataset_association_id + == LibraryDatasetDatasetAssociation.id) + & (not_(LibraryDatasetDatasetInfoAssociation.deleted)) + ), + backref="info_association") + + template = relationship('FormDefinition', + primaryjoin=(lambda: + LibraryDatasetDatasetInfoAssociation.form_definition_id == FormDefinition.id)) # type: ignore + info = relationship('FormValues', + primaryjoin=(lambda: + LibraryDatasetDatasetInfoAssociation.form_values_id == FormValues.id)) # type: ignore + def __init__(self, library_dataset_dataset_association, form_definition, info): # TODO: need to figure out if this should be inheritable to the associated LibraryDataset self.library_dataset_dataset_association = library_dataset_dataset_association diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6f681461999..c384e49f11a 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -151,15 +151,6 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), Column("message", TrimmedString(255))) -model.LibraryDatasetDatasetInfoAssociation.table = Table( - "library_dataset_dataset_info_association", metadata, - Column("id", Integer, primary_key=True), - Column("library_dataset_dataset_association_id", Integer, - ForeignKey("library_dataset_dataset_association.id"), nullable=True, index=True), - Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True), - Column("form_values_id", Integer, ForeignKey("form_values.id"), index=True), - Column("deleted", Boolean, index=True, default=False)) - model.Job.table = Table( "job", metadata, Column("id", Integer, primary_key=True), @@ -379,20 +370,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.L back_populates='dataset_ldda'), )) -mapper_registry.map_imperatively(model.LibraryDatasetDatasetInfoAssociation, model.LibraryDatasetDatasetInfoAssociation.table, properties=dict( - library_dataset_dataset_association=relation(model.LibraryDatasetDatasetAssociation, - primaryjoin=( - (model.LibraryDatasetDatasetInfoAssociation.table.c.library_dataset_dataset_association_id - == model.LibraryDatasetDatasetAssociation.table.c.id) - & (not_(model.LibraryDatasetDatasetInfoAssociation.table.c.deleted)) - ), - backref="info_association"), - template=relation(model.FormDefinition, - primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_definition_id == model.FormDefinition.id)), - info=relation(model.FormValues, - primaryjoin=(model.LibraryDatasetDatasetInfoAssociation.table.c.form_values_id == model.FormValues.id)) -)) - # simple_mapping( # model.ImplicitCollectionJobsHistoryDatasetCollectionAssociation, # history_dataset_collection_associations=relation( diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index c44950f8ea9..0d586a814a0 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -2885,6 +2885,48 @@ class TestLibraryDatasetDatasetAssociationTagAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestLibraryDatasetDatasetInfoAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset_dataset_info_association' + + def test_columns( + self, + session, + cls_, + library_dataset_dataset_association, + form_definition, + form_values, + ): + deleted = True + obj = cls_(library_dataset_dataset_association, form_definition, form_values) + obj.deleted = deleted + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert stored_obj.form_definition_id == form_definition.id + assert stored_obj.form_values_id == form_values.id + assert stored_obj.deleted == deleted + + def test_relationships( + self, + session, + cls_, + library_dataset_dataset_association, + form_definition, + form_values, + ): + obj = cls_(library_dataset_dataset_association, form_definition, form_values) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert stored_obj.template.id == form_definition.id + assert stored_obj.info.id == form_values.id + + class TestLibraryDatasetPermissions(BaseTest): def test_table(self, cls_): From a83ccb0a77b63e3c60899ee0f0a4875cea72fd8f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 12 Aug 2021 00:55:59 -0400 Subject: [PATCH 179/225] Add unit tests for LDDA --- lib/galaxy/model/__init__.py | 1 + test/unit/model/test_model.py | 174 ++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 4804009cf8d..082362d76df 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4702,6 +4702,7 @@ class LibraryDataset(Base, RepresentById): class LibraryDatasetDatasetAssociation(DatasetInstance, HasName, RepresentById): + def __init__(self, copied_from_history_dataset_association=None, copied_from_library_dataset_dataset_association=None, diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 0d586a814a0..eee52ff7834 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -2822,6 +2822,157 @@ class TestLibraryDatasetCollectionRatingAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestLibraryDatasetDatasetAssociation(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'library_dataset_dataset_association' + + def test_columns( + self, + session, + cls_, + library_dataset, + dataset, + history_dataset_association, + library_dataset_dataset_association_factory, + extended_metadata, + user, + model, + ): + create_time = datetime.now() + state = 'a' + name = 'b' + info = 'c' + blurb = 'd' + peek = 'e' + tool_version = 'f' + extension = 'g' + designation = 'i' + deleted = True + validated_state = 'j' + validated_state_message = 'k' + visible = True + message = 'm' + _metadata = 'n' + copied_from_ldda = library_dataset_dataset_association_factory() + parent = library_dataset_dataset_association_factory() + persist(session, copied_from_ldda) + persist(session, parent) + + obj = cls_() + obj.library_dataset = library_dataset + obj.dataset = dataset + obj.create_time = create_time + obj.copied_from_history_dataset_association = history_dataset_association + obj.copied_from_library_dataset_dataset_association = copied_from_ldda + obj.state = state + obj.name = name + obj.info = info + obj.blurb = blurb + obj.peek = peek + obj.tool_version = tool_version + obj.extension = extension + obj.parent_id = parent.id + obj.designation = designation + obj.deleted = deleted + obj.validated_state = validated_state + obj.validated_state_message = validated_state_message + obj.visible = visible + obj.extended_metadata = extended_metadata + obj.user = user + obj.message = message + obj._metadata = _metadata + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.library_dataset_id == library_dataset.id + assert stored_obj.dataset_id == dataset.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time >= create_time # this is sufficient for testing + assert stored_obj.copied_from_history_dataset_association_id == history_dataset_association.id + assert stored_obj.copied_from_library_dataset_dataset_association_id == copied_from_ldda.id + assert stored_obj.state == state + assert stored_obj.name == name + assert stored_obj.info == info + assert stored_obj.blurb == blurb + assert stored_obj.peek == peek + assert stored_obj.tool_version == tool_version + assert stored_obj.extension == extension + assert stored_obj.parent_id == parent.id + assert stored_obj.designation == designation + assert stored_obj.deleted == deleted + assert stored_obj.validated_state == validated_state + assert stored_obj.validated_state_message == validated_state_message + assert stored_obj.visible == visible + assert stored_obj.extended_metadata_id == extended_metadata.id + assert stored_obj.user_id == user.id + assert stored_obj.message == message + # We cannot test obj.metadata by setting it directly (like the other attributes). + # However, the following assertion verifies that it exists and has been initialized correctly. + assert stored_obj.metadata.parent.id == obj_id + assert stored_obj._metadata == _metadata + + delete_from_database(session, [copied_from_ldda, parent]) + + def test_relationships( + self, + session, + cls_, + library_dataset, + dataset, + history_dataset_association, + library_dataset_dataset_association_factory, + extended_metadata, + user, + library_dataset_dataset_association_tag_association, + library_dataset_dataset_association_permission, + history_dataset_association_factory, + implicitly_converted_dataset_association_factory, + job_to_input_library_dataset_association, + job_to_output_library_dataset_association, + ): + copied_from_ldda = library_dataset_dataset_association_factory() + parent = library_dataset_dataset_association_factory() + copied_to_hda = history_dataset_association_factory() + icda = implicitly_converted_dataset_association_factory() + icpda = implicitly_converted_dataset_association_factory() + persisted = [copied_from_ldda, parent, copied_to_hda, icda, icpda] + + obj = cls_() + obj.library_dataset = library_dataset + obj.dataset = dataset + obj.copied_from_history_dataset_association = history_dataset_association + obj.copied_from_library_dataset_dataset_association = copied_from_ldda + obj.extended_metadata = extended_metadata + obj.user = user + obj.tags.append(library_dataset_dataset_association_tag_association) + obj.actions.append(library_dataset_dataset_association_permission) + obj.copied_to_history_dataset_associations.append(copied_to_hda) + obj.implicitly_converted_datasets.append(icda) + obj.implicitly_converted_parent_datasets.append(icpda) + obj.dependent_jobs.append(job_to_input_library_dataset_association) + obj.creating_job_associations.append(job_to_output_library_dataset_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library_dataset.id == library_dataset.id + assert stored_obj.dataset.id == dataset.id + assert stored_obj.copied_from_history_dataset_association.id == history_dataset_association.id + assert stored_obj.copied_from_library_dataset_dataset_association.id == copied_from_ldda.id + assert stored_obj.extended_metadata.id == extended_metadata.id + assert stored_obj.user.id == user.id + assert stored_obj.tags == [library_dataset_dataset_association_tag_association] + assert stored_obj.actions == [library_dataset_dataset_association_permission] + assert stored_obj.copied_to_history_dataset_associations == [copied_to_hda] + assert stored_obj.implicitly_converted_datasets == [icda] + assert stored_obj.implicitly_converted_parent_datasets == [icpda] + assert stored_obj.dependent_jobs == [job_to_input_library_dataset_association] + assert stored_obj.creating_job_associations == [job_to_output_library_dataset_association] + + delete_from_database(session, persisted) + + class TestLibraryDatasetDatasetAssociationPermissions(BaseTest): def test_table(self, cls_): @@ -5989,12 +6140,24 @@ def job_to_implicit_output_dataset_collection_association(model, session, datase yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def job_to_input_library_dataset_association(model, session, library_dataset_dataset_association): + instance = model.JobToInputLibraryDatasetAssociation(None, library_dataset_dataset_association) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def job_to_output_dataset_collection_association(model, session, history_dataset_collection_association): instance = model.JobToOutputDatasetCollectionAssociation(None, history_dataset_collection_association) yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def job_to_output_library_dataset_association(model, session, library_dataset_dataset_association): + instance = model.JobToOutputLibraryDatasetAssociation(None, library_dataset_dataset_association) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def library(model, session): lb = model.Library() @@ -6424,6 +6587,17 @@ def history_rating_association_factory(model, user, history): return make_instance +@pytest.fixture +def implicitly_converted_dataset_association_factory(model, user, history_dataset_association): + def make_instance(*args, **kwds): + instance = model.ImplicitlyConvertedDatasetAssociation( + dataset=history_dataset_association, + parent=history_dataset_association, # using the same dataset; should work here. + ) + return instance + return make_instance + + @pytest.fixture def library_dataset_dataset_association_factory(model): def make_instance(*args, **kwds): From ce60446b18460cb5a076bfc39d7da3d5b897b4e2 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 12 Aug 2021 14:46:28 -0400 Subject: [PATCH 180/225] Add unit tests for User model --- test/unit/model/test_model.py | 223 +++++++++++++++++++++++++++++++++- 1 file changed, 221 insertions(+), 2 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index eee52ff7834..7aa17aae928 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -4370,6 +4370,152 @@ class TestTransferJob(BaseTest): assert stored_obj.transfer_job == [genome_index_tool_data] +class TestUser(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'user' + + def test_columns(self, session, cls_, form_values): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + email = get_random_string() + username = get_random_string() + password = 'c' + last_password_change = update_time + external = True + deleted = True + purged = True + disk_usage = 1 + active = False + activation_token = 'd' + + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.email = email + obj.username = username + obj.password = password + obj.last_password_change = last_password_change + obj.external = external + obj.values = form_values + obj.deleted = deleted + obj.purged = purged + obj.disk_usage = disk_usage + obj.active = active + obj.activation_token = activation_token + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.email == email + assert stored_obj.username == username + assert stored_obj.password == password + assert stored_obj.last_password_change == last_password_change + assert stored_obj.external == external + assert stored_obj.form_values_id == form_values.id + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + assert stored_obj.disk_usage == disk_usage + assert stored_obj.active == active + assert stored_obj.activation_token == activation_token + + def test_relationships( + self, + session, + cls_, + form_values, + user_address, + cloud_authz, + custos_authnz_token, + default_user_permissions, + user_group_association, + history_factory, + galaxy_session, + page_user_share_association, + user_quota_association, + user_authnz_token, + user_preference, + api_keys, + page, + password_reset_token, + history_user_share_association, + data_manager_history_association, + stored_workflow_user_share_association, + user_role_association, + stored_workflow, + stored_workflow_menu_entry_factory, + ): + history1 = history_factory(deleted=False) + history2 = history_factory(deleted=True) + + obj = cls_() + + obj.email = get_random_string() + obj.username = get_random_string() + obj.password = 'a' + obj.values = form_values + obj.addresses.append(user_address) + obj.cloudauthz.append(cloud_authz) + obj.custos_auth.append(custos_authnz_token) + obj.default_permissions.append(default_user_permissions) + obj.groups.append(user_group_association) + obj.histories.append(history1) + obj.histories.append(history2) + obj.galaxy_sessions.append(galaxy_session) + obj.pages_shared_by_others.append(page_user_share_association) + obj.quotas.append(user_quota_association) + obj.social_auth.append(user_authnz_token) + + swme = stored_workflow_menu_entry_factory() + swme.stored_workflow = stored_workflow + swme.user = obj + + user_preference.name = 'a' + obj._preferences.set(user_preference) + + obj.api_keys.append(api_keys) + obj.pages.append(page) + obj.reset_tokens.append(password_reset_token) + obj.histories_shared_by_others.append(history_user_share_association) + obj.data_manager_histories.append(data_manager_history_association) + obj.workflows_shared_by_others.append(stored_workflow_user_share_association) + obj.roles.append(user_role_association) + obj.stored_workflows.append(stored_workflow) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.values.id == form_values.id + assert stored_obj.addresses == [user_address] + assert stored_obj.cloudauthz == [cloud_authz] + assert stored_obj.custos_auth == [custos_authnz_token] + assert stored_obj.default_permissions == [default_user_permissions] + assert stored_obj.groups == [user_group_association] + + assert len(stored_obj.histories) == 2 + assert history1 in stored_obj.histories + assert history2 in stored_obj.histories + + assert stored_obj.active_histories == [history1] + assert stored_obj.galaxy_sessions == [galaxy_session] + assert stored_obj.pages_shared_by_others == [page_user_share_association] + assert stored_obj.quotas == [user_quota_association] + assert stored_obj.social_auth == [user_authnz_token] + assert stored_obj.stored_workflow_menu_entries == [swme] + assert user_preference in stored_obj._preferences.values() + assert stored_obj.api_keys == [api_keys] + assert stored_obj.pages == [page] + assert stored_obj.reset_tokens == [password_reset_token] + assert stored_obj.histories_shared_by_others == [history_user_share_association] + assert stored_obj.data_manager_histories == [data_manager_history_association] + assert stored_obj.workflows_shared_by_others == [stored_workflow_user_share_association] + assert stored_obj.roles == [user_role_association] + assert stored_obj.stored_workflows == [stored_workflow] + + delete_from_database(session, [history1, history2, swme]) + + class TestUserAction(BaseTest): def test_table(self, cls_): @@ -5876,6 +6022,12 @@ def session(model): # Fixtures yielding persisted instances of models, deleted from the database on test exit. +@pytest.fixture +def api_keys(model, session): + instance = model.APIKeys() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def cleanup_event(model, session): ce = model.CleanupEvent() @@ -5888,6 +6040,18 @@ def cloud_authz(model, session, user, user_authnz_token): yield from dbcleanup_wrapper(session, ca) +@pytest.fixture +def custos_authnz_token(model, session, user, user_authnz_token): + instance = model.CustosAuthnzToken(user, None, None, None, None, None, None, None) + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def data_manager_history_association(model, session): + instance = model.DataManagerHistoryAssociation() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def dataset(model, session): d = model.Dataset() @@ -5945,6 +6109,12 @@ def default_quota_association(model, session, quota): yield from dbcleanup_wrapper(session, dqa) +@pytest.fixture +def default_user_permissions(model, session, user, role): + instance = model.DefaultUserPermissions(user, None, role) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def deferred_job(model, session): dj = model.DeferredJob() @@ -6298,6 +6468,14 @@ def page_user_share_association(model, session): yield from dbcleanup_wrapper(session, pra) +@pytest.fixture +def password_reset_token(model, session, user): + token = get_random_string() + instance = model.PasswordResetToken(user, token) + where_clause = type(instance).token == token + yield from dbcleanup_wrapper(session, instance, where_clause) + + @pytest.fixture def post_job_action(model, session): pja = model.PostJobAction('a') @@ -6389,12 +6567,36 @@ def user(model, session): yield from dbcleanup_wrapper(session, u) +@pytest.fixture +def user_address(model, session, user): + instance = model.UserAddress() + instance.name = 'a' + instance.address = 'b' + instance.city = 'c' + instance.state = 'd' + instance.postal_code = 'e' + instance.country = 'f' + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def user_authnz_token(model, session, user): t = model.UserAuthnzToken('a', 'b', 'c', 1, 'd', user) yield from dbcleanup_wrapper(session, t) +@pytest.fixture +def user_group_association(model, session): + instance = model.UserGroupAssociation(None, None) + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def user_preference(model, session): + instance = model.UserPreference() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def user_quota_association(model, session): uqa = model.UserQuotaAssociation(None, None) @@ -6566,6 +6768,16 @@ def dataset_collection_factory(model): return make_instance +@pytest.fixture +def history_factory(model): + def make_instance(**kwds): + instance = model.History() + if 'deleted' in kwds: + instance.deleted = kwds['deleted'] + return instance + return make_instance + + @pytest.fixture def history_dataset_association_factory(model): def make_instance(*args, **kwds): @@ -6619,6 +6831,13 @@ def page_rating_association_factory(model): return make_instance +@pytest.fixture +def stored_workflow_menu_entry_factory(model): + def make_instance(*args, **kwds): + return model.StoredWorkflowMenuEntry(*args, **kwds) + return make_instance + + @pytest.fixture def stored_workflow_rating_association_factory(model): def make_instance(*args, **kwds): @@ -6691,8 +6910,8 @@ def workflow_step_factory(model, workflow): # Mics. helpers. -def dbcleanup_wrapper(session, obj): - with dbcleanup(session, obj): +def dbcleanup_wrapper(session, obj, where_clause=None): + with dbcleanup(session, obj, where_clause): yield obj From a5eb4cfbe1d24e9962617f3580b5dd257a83b2b0 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 12 Aug 2021 15:57:57 -0400 Subject: [PATCH 181/225] Import desc from sqlalchemy, not galaxy --- lib/galaxy/webapps/galaxy/controllers/workflow.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/workflow.py b/lib/galaxy/webapps/galaxy/controllers/workflow.py index c9f456047f6..e181a082456 100644 --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -5,7 +5,7 @@ from html.parser import HTMLParser from http.client import HTTPConnection from markupsafe import escape -from sqlalchemy import and_ +from sqlalchemy import and_, desc from sqlalchemy.orm import eagerload, joinedload, lazyload, undefer from sqlalchemy.sql import expression @@ -20,7 +20,6 @@ from galaxy.managers.workflows import ( WorkflowUpdateOptions, ) from galaxy.model.item_attrs import UsesItemRatings -from galaxy.model.mapping import desc from galaxy.security.validate_user_input import validate_publicname from galaxy.tools.parameters.basic import workflow_building_modes from galaxy.util import ( From 202e31d4cd15213d2a8f79695f32b9dbefce8e63 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 12 Aug 2021 18:01:11 -0400 Subject: [PATCH 182/225] Add unit tests for HistoryDatasetAssociation --- test/unit/model/test_model.py | 192 +++++++++++++++++++++++++++++++++- 1 file changed, 190 insertions(+), 2 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 7aa17aae928..68f4611714a 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1342,6 +1342,164 @@ class TestHistory(BaseTest): delete_from_database(session, to_cleanup) +class TestHistoryDatasetAssociation(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'history_dataset_association' + + def test_columns( + self, + session, + cls_, + history, + dataset, + library_dataset_dataset_association, + extended_metadata, + history_dataset_collection_association, + history_dataset_association_factory, + ): + copied_from_hda = history_dataset_association_factory() + parent = history_dataset_association_factory() + + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + state = 'a' + name = 'b' + info = 'c' + blurb = 'd' + peek = 'e' + tool_version = 'f' + extension = 'g' + _metadata = 'h' + designation = 'i' + deleted = False + visible = False + version = 1 + hid = 2 + purged = False + validated_state = 'j' + validated_state_message = 'k' + + obj = cls_() + obj.history = history + obj.dataset = dataset + obj.create_time = create_time + obj.update_time = update_time + obj.state = state + obj.copied_from_history_dataset_association = copied_from_hda + obj.copied_from_library_dataset_dataset_association = library_dataset_dataset_association + obj.name = name + obj.info = info + obj.blurb = blurb + obj.peek = peek + obj.tool_version = tool_version + obj.extension = extension + obj._metadata = _metadata + obj.parent_id = parent.id + obj.designation = designation + obj.deleted = deleted + obj.visible = visible + obj.extended_metadata = extended_metadata + obj.version = version + obj.hid = hid + obj.purged = purged + obj.validated_state = validated_state + obj.validated_state_message = validated_state_message + obj.hidden_beneath_collection_instance = history_dataset_collection_association + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_id == history.id + assert stored_obj.dataset_id == dataset.id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.state == state + assert stored_obj.copied_from_history_dataset_association_id == copied_from_hda.id + assert stored_obj.copied_from_library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert stored_obj.name == name + assert stored_obj.info == info + assert stored_obj.blurb == blurb + assert stored_obj.peek == peek + assert stored_obj.tool_version == tool_version + assert stored_obj.extension == extension + assert stored_obj._metadata == _metadata + assert stored_obj.parent_id == parent.id + assert stored_obj.designation == designation + assert stored_obj.deleted == deleted + assert stored_obj.visible == visible + assert stored_obj.extended_metadata_id == extended_metadata.id + assert stored_obj.version == version + assert stored_obj.hid == hid + assert stored_obj.purged == purged + assert stored_obj.validated_state == validated_state + assert stored_obj.validated_state_message == validated_state_message + assert stored_obj.hidden_beneath_collection_instance_id == history_dataset_collection_association.id + + delete_from_database(session, [copied_from_hda, parent]) + + def test_relationships( + self, + session, + cls_, + history, + dataset, + extended_metadata, + history_dataset_collection_association, + history_dataset_association_factory, + history_dataset_association_tag_association, + history_dataset_association_annotation_association, + history_dataset_association_rating_association, + job_to_input_dataset_association, + job_to_output_dataset_association, + implicitly_converted_dataset_association_factory, + library_dataset_dataset_association_factory, + ): + copied_from_hda = history_dataset_association_factory() + copied_from_ldda = library_dataset_dataset_association_factory() + copied_to_ldda = library_dataset_dataset_association_factory() + + icda = implicitly_converted_dataset_association_factory() + icpda = implicitly_converted_dataset_association_factory() + + obj = cls_() + obj.history = history + obj.dataset = dataset + obj.copied_from_history_dataset_association = copied_from_hda + obj.copied_from_library_dataset_dataset_association = copied_from_ldda + obj.extended_metadata = extended_metadata + obj.validated_state = 'a' + obj.hidden_beneath_collection_instance = history_dataset_collection_association + obj.copied_to_library_dataset_dataset_associations.append(copied_to_ldda) + obj.tags.append(history_dataset_association_tag_association) + obj.annotations.append(history_dataset_association_annotation_association) + obj.ratings.append(history_dataset_association_rating_association) + obj.dependent_jobs.append(job_to_input_dataset_association) + obj.creating_job_associations.append(job_to_output_dataset_association) + obj.implicitly_converted_datasets.append(icda) + obj.implicitly_converted_parent_datasets.append(icpda) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history.id == history.id + assert stored_obj.dataset.id == dataset.id + assert stored_obj.copied_from_history_dataset_association.id == copied_from_hda.id + assert stored_obj.copied_from_library_dataset_dataset_association.id == copied_from_ldda.id + assert stored_obj.extended_metadata.id == extended_metadata.id + assert stored_obj.hidden_beneath_collection_instance.id == history_dataset_collection_association.id + assert stored_obj.copied_to_library_dataset_dataset_associations == [copied_to_ldda] + assert stored_obj.tags == [history_dataset_association_tag_association] + assert stored_obj.annotations == [history_dataset_association_annotation_association] + assert stored_obj.ratings == [history_dataset_association_rating_association] + assert stored_obj.dependent_jobs == [job_to_input_dataset_association] + assert stored_obj.creating_job_associations == [job_to_output_dataset_association] + assert stored_obj.implicitly_converted_datasets == [icda] + assert stored_obj.implicitly_converted_parent_datasets == [icpda] + + delete_from_database(session, [copied_from_hda, copied_from_ldda, copied_to_ldda, icda, icpda]) + + class TestHistoryDatasetAssociationHistory(BaseTest): def test_table(self, cls_): @@ -6211,10 +6369,28 @@ def history_dataset_association(model, session, dataset): yield from dbcleanup_wrapper(session, hda) +@pytest.fixture +def history_dataset_annotation_association(model, session): + instance = model.HistoryDatasetAnnotationAssociation() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def history_dataset_association_annotation_association(model, session): + instance = model.HistoryDatasetAssociationAnnotationAssociation() + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def history_dataset_association_rating_association(model, session): + instance = model.HistoryDatasetAssociationRatingAssociation(None, None) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def history_dataset_association_tag_association(model, session): - hdata = model.HistoryDatasetAssociationTagAssociation() - yield from dbcleanup_wrapper(session, hdata) + instance = model.HistoryDatasetAssociationTagAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6310,6 +6486,12 @@ def job_to_implicit_output_dataset_collection_association(model, session, datase yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def job_to_input_dataset_association(model, session, history_dataset_association): + instance = model.JobToInputDatasetAssociation(None, history_dataset_association) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def job_to_input_library_dataset_association(model, session, library_dataset_dataset_association): instance = model.JobToInputLibraryDatasetAssociation(None, library_dataset_dataset_association) @@ -6322,6 +6504,12 @@ def job_to_output_dataset_collection_association(model, session, history_dataset yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def job_to_output_dataset_association(model, session, history_dataset_association): + instance = model.JobToOutputDatasetAssociation(None, history_dataset_association) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def job_to_output_library_dataset_association(model, session, library_dataset_dataset_association): instance = model.JobToOutputLibraryDatasetAssociation(None, library_dataset_dataset_association) From 70d3408255d2544d87d591cc254fdc15f7346bd6 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 12 Aug 2021 22:31:35 -0400 Subject: [PATCH 183/225] Add unit tests for Job --- test/unit/model/test_model.py | 269 ++++++++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 68f4611714a..9e447b9efd8 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -2254,6 +2254,209 @@ class TestInteractiveToolEntryPoint(BaseTest): assert stored_obj.job.id == job.id +class TestJob(BaseTest): + + # def test_table(self, cls_): + # assert cls_.__tablename__ == 'job' + + def test_columns( + self, + session, + cls_, + history, + library_folder, + dynamic_tool, + galaxy_session, + user, + ): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + tool_id = 'a' + tool_version = 'b' + galaxy_version = 'c' + state = 'd' + info = 'e' + copied_from_job_id = 1 + command_line = 'f' + dependencies = 'g' + job_messages = 'h' + param_filename = 'i' + runner_name = 'j' + job_stdout = 'k' + job_stderr = 'm' + tool_stdout = 'n' + tool_stderr = 'o' + exit_code = 2 + traceback = 'p' + job_runner_name = 'q' + job_runner_external_id = 'r' + destination_id = 's' + destination_params = 'u' + object_store_id = 'v' + imported = False + params = 'w' + handler = 'x' + + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.history = history + obj.library_folder = library_folder + obj.tool_id = tool_id + obj.tool_version = tool_version + obj.galaxy_version = galaxy_version + obj.dynamic_tool_id = dynamic_tool.id + obj.state = state + obj.info = info + obj.copied_from_job_id = copied_from_job_id + obj.command_line = command_line + obj.dependencies = dependencies + obj.job_messages = job_messages + obj.param_filename = param_filename + obj.runner_name = runner_name + obj.job_stdout = job_stdout + obj.job_stderr = job_stderr + obj.tool_stdout = tool_stdout + obj.tool_stderr = tool_stderr + obj.exit_code = exit_code + obj.traceback = traceback + obj.galaxy_session = galaxy_session + obj.user = user + obj.job_runner_name = job_runner_name + obj.job_runner_external_id = job_runner_external_id + obj.destination_id = destination_id + obj.destination_params = destination_params + obj.object_store_id = object_store_id + obj.imported = imported + obj.params = params + obj.handler = handler + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id, unique=True) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.history_id == history.id + assert stored_obj.library_folder_id == library_folder.id + assert stored_obj.tool_id == tool_id + assert stored_obj.tool_version == tool_version + assert stored_obj.galaxy_version == galaxy_version + assert stored_obj.dynamic_tool_id == dynamic_tool.id + assert stored_obj.state == state + assert stored_obj.info == info + assert stored_obj.copied_from_job_id == copied_from_job_id + assert stored_obj.command_line == command_line + assert stored_obj.dependencies == dependencies + assert stored_obj.job_messages == job_messages + assert stored_obj.param_filename == param_filename + assert stored_obj.runner_name == runner_name + assert stored_obj.job_stdout == job_stdout + assert stored_obj.job_stderr == job_stderr + assert stored_obj.tool_stdout == tool_stdout + assert stored_obj.tool_stderr == tool_stderr + assert stored_obj.exit_code == exit_code + assert stored_obj.traceback == traceback + assert stored_obj.session_id == galaxy_session.id + assert stored_obj.user_id == user.id + assert stored_obj.job_runner_name == job_runner_name + assert stored_obj.job_runner_external_id == job_runner_external_id + assert stored_obj.destination_id == destination_id + assert stored_obj.destination_params == destination_params + assert stored_obj.object_store_id == object_store_id + assert stored_obj.imported == imported + assert stored_obj.params == params + assert stored_obj.handler == handler + + def test_relationships( + self, + session, + cls_, + history, + library_folder, + dynamic_tool, + galaxy_session, + user, + job_parameter, + job_to_input_dataset_association, + job_to_input_dataset_collection_association, + job_to_input_dataset_collection_element_association, + job_to_output_dataset_collection_association, + job_to_implicit_output_dataset_collection_association, + post_job_action_association, + job_to_input_library_dataset_association, + job_to_output_library_dataset_association, + job_external_output_metadata, + task, + job_to_output_dataset_association, + job_state_history, + job_metric_text, + job_metric_numeric, + genome_index_tool_data, + interactive_tool_entry_point, + implicit_collection_jobs_job_association, + job_container_association, + data_manager_job_association, + history_dataset_collection_association, + workflow_invocation_step, + ): + obj = cls_() + obj.history = history + obj.library_folder = library_folder + obj.galaxy_session = galaxy_session + obj.user = user + obj.parameters.append(job_parameter) + obj.input_datasets.append(job_to_input_dataset_association) + obj.input_dataset_collections.append(job_to_input_dataset_collection_association) + obj.input_dataset_collection_elements.append(job_to_input_dataset_collection_element_association) + obj.output_dataset_collection_instances.append(job_to_output_dataset_collection_association) + obj.output_dataset_collections.append(job_to_implicit_output_dataset_collection_association) + obj.post_job_actions.append(post_job_action_association) + obj.input_library_datasets.append(job_to_input_library_dataset_association) + obj.output_library_datasets.append(job_to_output_library_dataset_association) + obj.external_output_metadata.append(job_external_output_metadata) + obj.tasks.append(task) + obj.output_datasets.append(job_to_output_dataset_association) + obj.state_history.append(job_state_history) + obj.text_metrics.append(job_metric_text) + obj.numeric_metrics.append(job_metric_numeric) + obj.job.append(genome_index_tool_data) + obj.interactivetool_entry_points.append(interactive_tool_entry_point) + obj.implicit_collection_jobs_association = implicit_collection_jobs_job_association + obj.container = job_container_association + obj.data_manager_association = data_manager_job_association + obj.history_dataset_collection_associations.append(history_dataset_collection_association) + obj.workflow_invocation_step = workflow_invocation_step + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id, unique=True) + assert stored_obj.history_id == history.id + assert stored_obj.library_folder_id == library_folder.id + assert stored_obj.session_id == galaxy_session.id + assert stored_obj.user_id == user.id + assert stored_obj.parameters == [job_parameter] + assert stored_obj.input_datasets == [job_to_input_dataset_association] + assert stored_obj.input_dataset_collections == [job_to_input_dataset_collection_association] + assert stored_obj.input_dataset_collection_elements == [job_to_input_dataset_collection_element_association] + assert stored_obj.output_dataset_collection_instances == [job_to_output_dataset_collection_association] + assert stored_obj.output_dataset_collections == [job_to_implicit_output_dataset_collection_association] + assert stored_obj.post_job_actions == [post_job_action_association] + assert stored_obj.input_library_datasets == [job_to_input_library_dataset_association] + assert stored_obj.output_library_datasets == [job_to_output_library_dataset_association] + assert stored_obj.external_output_metadata == [job_external_output_metadata] + assert stored_obj.tasks == [task] + assert stored_obj.output_datasets == [job_to_output_dataset_association] + assert job_state_history in stored_obj.state_history # sufficient for test + assert stored_obj.text_metrics == [job_metric_text] + assert stored_obj.numeric_metrics == [job_metric_numeric] + assert stored_obj.job == [genome_index_tool_data] + assert stored_obj.interactivetool_entry_points == [interactive_tool_entry_point] + assert stored_obj.implicit_collection_jobs_association == implicit_collection_jobs_job_association + assert stored_obj.container == job_container_association + assert stored_obj.data_manager_association == data_manager_job_association + assert stored_obj.history_dataset_collection_associations == [history_dataset_collection_association] + assert stored_obj.workflow_invocation_step == workflow_invocation_step + + class TestJobContainerAssociation(BaseTest): def test_table(self, cls_): @@ -6210,6 +6413,12 @@ def data_manager_history_association(model, session): yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def data_manager_job_association(model, session): + instance = model.DataManagerJobAssociation() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def dataset(model, session): d = model.Dataset() @@ -6468,18 +6677,60 @@ def implicitly_created_dataset_collection_input(model, session, history_dataset_ yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def interactive_tool_entry_point(model, session): + instance = model.InteractiveToolEntryPoint() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def job(model, session): j = model.Job() yield from dbcleanup_wrapper(session, j) +@pytest.fixture +def job_container_association(model, session): + instance = model.JobContainerAssociation() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def job_export_history_archive(model, session): jeha = model.JobExportHistoryArchive() yield from dbcleanup_wrapper(session, jeha) +@pytest.fixture +def job_external_output_metadata(model, session, job, history_dataset_association): + instance = model.JobExternalOutputMetadata(job, history_dataset_association) + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def job_metric_numeric(model, session): + instance = model.JobMetricNumeric(None, None, None) + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def job_metric_text(model, session): + instance = model.JobMetricText(None, None, None) + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def job_parameter(model, session): + instance = model.JobParameter(None, None) + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def job_state_history(model, session, job): + instance = model.JobStateHistory(job) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def job_to_implicit_output_dataset_collection_association(model, session, dataset_collection): instance = model.JobToImplicitOutputDatasetCollectionAssociation(None, dataset_collection) @@ -6492,6 +6743,18 @@ def job_to_input_dataset_association(model, session, history_dataset_association yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def job_to_input_dataset_collection_association(model, session, history_dataset_collection_association): + instance = model.JobToInputDatasetCollectionAssociation(None, history_dataset_collection_association) + yield from dbcleanup_wrapper(session, instance) + + +@pytest.fixture +def job_to_input_dataset_collection_element_association(model, session, dataset_collection_element): + instance = model.JobToInputDatasetCollectionElementAssociation(None, dataset_collection_element) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def job_to_input_library_dataset_association(model, session, library_dataset_dataset_association): instance = model.JobToInputLibraryDatasetAssociation(None, library_dataset_dataset_association) @@ -6670,6 +6933,12 @@ def post_job_action(model, session): yield from dbcleanup_wrapper(session, pja) +@pytest.fixture +def post_job_action_association(model, session, post_job_action, job): + instance = model.PostJobActionAssociation(post_job_action, job) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def quota(model, session): q = model.Quota(get_random_string(), 'b') From 9c55e3a901be7f62d61ecd2ff3777b53a2b3f93c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 12 Aug 2021 22:52:40 -0400 Subject: [PATCH 184/225] Map Job declaratively --- lib/galaxy/model/__init__.py | 82 +++++++++++++++++++++++++++++++++-- lib/galaxy/model/mapping.py | 79 ++------------------------------- test/unit/model/test_model.py | 4 +- 3 files changed, 84 insertions(+), 81 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 082362d76df..2140423512a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -794,14 +794,88 @@ class TaskMetricNumeric(Base, BaseJobMetric, RepresentById): task = relationship('Task', back_populates='numeric_metrics') -class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): - dict_collection_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version'] - dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version', 'command_version'] - +class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): """ A job represents a request to run a tool given input datasets, tool parameters, and output datasets. """ + __tablename__ = 'job' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now, index=True) + history_id = Column(Integer, ForeignKey('history.id'), index=True) + library_folder_id = Column(Integer, ForeignKey('library_folder.id'), index=True) + tool_id = Column(String(255)) + tool_version = Column(TEXT, default='1.0.0') + galaxy_version = Column(String(64), default=None) + dynamic_tool_id = Column(Integer, ForeignKey('dynamic_tool.id'), index=True, nullable=True) + state = Column(String(64), index=True) + info = Column(TrimmedString(255)) + copied_from_job_id = Column(Integer, nullable=True) + command_line = Column(TEXT) + dependencies = Column(MutableJSONType, nullable=True) + job_messages = Column(MutableJSONType, nullable=True) + param_filename = Column(String(1024)) + runner_name = Column(String(255)) + job_stdout = Column(TEXT) + job_stderr = Column(TEXT) + tool_stdout = Column(TEXT) + tool_stderr = Column(TEXT) + exit_code = Column(Integer, nullable=True) + traceback = Column(TEXT) + session_id = Column(Integer, ForeignKey('galaxy_session.id'), index=True, nullable=True) + user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=True) + job_runner_name = Column(String(255)) + job_runner_external_id = Column(String(255), index=True) + destination_id = Column(String(255), nullable=True) + destination_params = Column(MutableJSONType, nullable=True) + object_store_id = Column(TrimmedString(255), index=True) + imported = Column(Boolean, default=False, index=True) + params = Column(TrimmedString(255), index=True) + handler = Column(TrimmedString(255), index=True) + + user = relationship('User') + galaxy_session = relationship('GalaxySession') + history = relationship('History', backref='jobs') + library_folder = relationship('LibraryFolder', lazy=True) + parameters = relationship('JobParameter', lazy=True) + input_datasets = relationship('JobToInputDatasetAssociation', back_populates='job') + input_dataset_collections = relationship('JobToInputDatasetCollectionAssociation', + back_populates='job', lazy=True) + input_dataset_collection_elements = relationship('JobToInputDatasetCollectionElementAssociation', + back_populates='job', lazy=True) + output_dataset_collection_instances = relationship('JobToOutputDatasetCollectionAssociation', + back_populates='job', lazy=True) + output_dataset_collections = relationship('JobToImplicitOutputDatasetCollectionAssociation', + back_populates='job', lazy=True) + post_job_actions = relationship('PostJobActionAssociation', backref='job', lazy=False) + input_library_datasets = relationship('JobToInputLibraryDatasetAssociation', + back_populates='job') + output_library_datasets = relationship('JobToOutputLibraryDatasetAssociation', + back_populates='job', lazy=True) + external_output_metadata = relationship('JobExternalOutputMetadata', lazy=True, backref='job') + tasks = relationship('Task', back_populates='job') + output_datasets = relationship('JobToOutputDatasetAssociation', back_populates='job') + state_history = relationship('JobStateHistory', back_populates='job') + text_metrics = relationship('JobMetricText', back_populates='job') + numeric_metrics = relationship('JobMetricNumeric', back_populates='job') + job = relationship('GenomeIndexToolData', back_populates='job') # TODO review attr naming (the functionality IS correct) + interactivetool_entry_points = relationship('InteractiveToolEntryPoint', + back_populates='job', uselist=True) + implicit_collection_jobs_association = relationship('ImplicitCollectionJobsJobAssociation', + back_populates='job', uselist=False) + container = relationship('JobContainerAssociation', back_populates='job', uselist=False) + data_manager_association = relationship('DataManagerJobAssociation', + back_populates='job', uselist=False) + history_dataset_collection_associations = relationship('HistoryDatasetCollectionAssociation', + back_populates='job') + workflow_invocation_step = relationship('WorkflowInvocationStep', + back_populates='job', uselist=False) + + dict_collection_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version'] + dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version', 'command_version'] + _numeric_metric = JobMetricNumeric _text_metric = JobMetricText diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index c384e49f11a..a79c6e8e150 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -19,7 +19,7 @@ from sqlalchemy import ( not_, Numeric, select, - String, Table, + Table, TEXT, true, ) @@ -34,7 +34,6 @@ from galaxy.model import mapper_registry from galaxy.model.base import SharedModelMapping from galaxy.model.custom_types import ( JSONType, - MutableJSONType, TrimmedString, UUIDType, ) @@ -151,42 +150,6 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), Column("message", TrimmedString(255))) -model.Job.table = Table( - "job", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now, index=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("library_folder_id", Integer, ForeignKey("library_folder.id"), index=True), - Column("tool_id", String(255)), - Column("tool_version", TEXT, default="1.0.0"), - Column("galaxy_version", String(64), default=None), - Column("dynamic_tool_id", Integer, ForeignKey("dynamic_tool.id"), index=True, nullable=True), - Column("state", String(64), index=True), - Column("info", TrimmedString(255)), - Column("copied_from_job_id", Integer, nullable=True), - Column("command_line", TEXT), - Column("dependencies", MutableJSONType, nullable=True), - Column("job_messages", MutableJSONType, nullable=True), - Column("param_filename", String(1024)), - Column("runner_name", String(255)), - Column("job_stdout", TEXT), - Column("job_stderr", TEXT), - Column("tool_stdout", TEXT), - Column("tool_stderr", TEXT), - Column("exit_code", Integer, nullable=True), - Column("traceback", TEXT), - Column("session_id", Integer, ForeignKey("galaxy_session.id"), index=True, nullable=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True, nullable=True), - Column("job_runner_name", String(255)), - Column("job_runner_external_id", String(255), index=True), - Column("destination_id", String(255), nullable=True), - Column("destination_params", MutableJSONType, nullable=True), - Column("object_store_id", TrimmedString(255), index=True), - Column("imported", Boolean, default=False, index=True), - Column("params", TrimmedString(255), index=True), - Column("handler", TrimmedString(255), index=True)) - # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -244,7 +207,7 @@ simple_mapping(model.HistoryDatasetAssociation, simple_mapping(model.Dataset, actions=relation(model.DatasetPermissions, back_populates='dataset'), - job=relation(model.Job, primaryjoin=(model.Dataset.table.c.job_id == model.Job.table.c.id)), + job=relation(model.Job, primaryjoin=(model.Dataset.table.c.job_id == model.Job.id)), active_history_associations=relation(model.HistoryDatasetAssociation, primaryjoin=( (model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id) @@ -394,50 +357,16 @@ model.Page.users_shared_with_dot_users = association_proxy('users_shared_with', # returns a list of users that visualization is shared with. model.Visualization.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore -mapper_registry.map_imperatively(model.Job, model.Job.table, properties=dict( - # user=relation( model.User.mapper ), - user=relation(model.User), - galaxy_session=relation(model.GalaxySession), - history=relation(model.History, backref="jobs"), - library_folder=relation(model.LibraryFolder, lazy=True), - parameters=relation(model.JobParameter, lazy=True), - input_datasets=relation(model.JobToInputDatasetAssociation, back_populates="job"), - input_dataset_collections=relation(model.JobToInputDatasetCollectionAssociation, back_populates="job", lazy=True), - input_dataset_collection_elements=relation(model.JobToInputDatasetCollectionElementAssociation, - back_populates="job", lazy=True), - output_dataset_collection_instances=relation(model.JobToOutputDatasetCollectionAssociation, - back_populates="job", lazy=True), - output_dataset_collections=relation(model.JobToImplicitOutputDatasetCollectionAssociation, - back_populates="job", lazy=True), - post_job_actions=relation(model.PostJobActionAssociation, backref="job", lazy=False), - input_library_datasets=relation(model.JobToInputLibraryDatasetAssociation, back_populates="job"), - output_library_datasets=relation(model.JobToOutputLibraryDatasetAssociation, - back_populates="job", lazy=True), - external_output_metadata=relation(model.JobExternalOutputMetadata, lazy=True, backref='job'), - tasks=relation(model.Task, back_populates='job'), - output_datasets=relation(model.JobToOutputDatasetAssociation, back_populates='job'), - state_history=relation(model.JobStateHistory, back_populates='job'), - text_metrics=relation(model.JobMetricText, back_populates='job'), - numeric_metrics=relation(model.JobMetricNumeric, back_populates='job'), - job=relation(model.GenomeIndexToolData, back_populates='job'), # TODO review attr naming (the functionality IS correct) - interactivetool_entry_points=relation(model.InteractiveToolEntryPoint, back_populates='job', uselist=True), - implicit_collection_jobs_association=relation(model.ImplicitCollectionJobsJobAssociation, - back_populates='job', uselist=False), - container=relation('JobContainerAssociation', back_populates='job', uselist=False), - data_manager_association=relation('DataManagerJobAssociation', back_populates='job', uselist=False), - history_dataset_collection_associations=relation('HistoryDatasetCollectionAssociation', back_populates='job'), - workflow_invocation_step=relation('WorkflowInvocationStep', back_populates='job', uselist=False), -)) model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], - and_(model.Job.table.c.id == model.JobToOutputDatasetAssociation.job_id, + and_(model.Job.id == model.JobToOutputDatasetAssociation.job_id, model.HistoryDatasetAssociation.table.c.id == model.JobToOutputDatasetAssociation.dataset_id, model.HistoryDatasetAssociation.table.c.deleted == true()) ) ) model.Job.any_output_dataset_collection_instances_deleted = column_property( # type: ignore exists([model.HistoryDatasetCollectionAssociation.id], - and_(model.Job.table.c.id == model.JobToOutputDatasetCollectionAssociation.job_id, + and_(model.Job.id == model.JobToOutputDatasetCollectionAssociation.job_id, model.HistoryDatasetCollectionAssociation.id == model.JobToOutputDatasetCollectionAssociation.dataset_collection_id, model.HistoryDatasetCollectionAssociation.deleted == true()) ) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 9e447b9efd8..f34ce5b1634 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -2256,8 +2256,8 @@ class TestInteractiveToolEntryPoint(BaseTest): class TestJob(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'job' + def test_table(self, cls_): + assert cls_.__tablename__ == 'job' def test_columns( self, From 54ac518e3bda96dc71340b0f42dd88a32eb83c3e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 13 Aug 2021 17:18:05 -0400 Subject: [PATCH 185/225] Map User declaratively (see note) This kills EVERYTHING. But the subsequent commit should fix it. --- lib/galaxy/model/__init__.py | 80 ++++++++++++++++++++-- lib/galaxy/model/mapping.py | 124 ++++++++++++++-------------------- test/unit/model/test_model.py | 4 +- 3 files changed, 127 insertions(+), 81 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 2140423512a..0ee6b646527 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -64,6 +64,7 @@ from sqlalchemy import ( ) from sqlalchemy.exc import OperationalError from sqlalchemy.ext import hybrid +from sqlalchemy.ext.orderinglist import ordering_list from sqlalchemy.orm import ( aliased, column_property, @@ -74,6 +75,7 @@ from sqlalchemy.orm import ( registry, relationship, ) +from sqlalchemy.orm.collections import attribute_mapped_collection from sqlalchemy.orm.decl_api import DeclarativeMeta import galaxy.exceptions @@ -425,14 +427,80 @@ class JobLike: raise NotImplementedError("Attempt to set stdout, must set tool_stderr or job_stderr") -class User(Dictifiable, RepresentById): +class User(Base, Dictifiable, RepresentById): use_pbkdf2 = True bootstrap_admin_user = False - api_keys: 'List[APIKeys]' + # api_keys: 'List[APIKeys]' already declared as relationship() """ Data for a Galaxy user or admin and relations to their histories, credentials, and roles. """ + + __tablename__ = 'galaxy_user' + + id = Column(Integer, primary_key=True) + create_time = Column(DateTime, default=now) + update_time = Column(DateTime, default=now, onupdate=now) + email = Column(TrimmedString(255), index=True, nullable=False) + username = Column(TrimmedString(255), index=True, unique=True) + password = Column(TrimmedString(255), nullable=False) + last_password_change = Column(DateTime, default=now) + external = Column(Boolean, default=False) + form_values_id = Column(Integer, ForeignKey('form_values.id'), index=True) + deleted = Column(Boolean, index=True, default=False) + purged = Column(Boolean, index=True, default=False) + disk_usage = Column(Numeric(15, 0), index=True) + # Column("person_metadata", JSONType), # TODO: add persistent, configurable metadata rep for workflow creator + active = Column(Boolean, index=True, default=True, nullable=False) + activation_token = Column(TrimmedString(64), nullable=True, index=True) + + addresses = relationship('UserAddress', + back_populates='user', + order_by='desc(UserAddress.update_time)') + cloudauthz = relationship('CloudAuthz', back_populates='user') + custos_auth = relationship('CustosAuthnzToken', back_populates='user') + default_permissions = relationship('DefaultUserPermissions', back_populates='user') + groups = relationship('UserGroupAssociation', back_populates='user') + histories = relationship('History', + backref='user', + order_by='desc(History.update_time)') + active_histories = relationship('History', + primaryjoin=(lambda: (History.user_id == User.id) & (not_(History.deleted))), # type: ignore + viewonly=True, + order_by='desc(History.update_time)') + galaxy_sessions = relationship('GalaxySession', + backref='user', + order_by='desc(GalaxySession.update_time)') + pages_shared_by_others = relationship('PageUserShareAssociation', back_populates='user') + quotas = relationship('UserQuotaAssociation', back_populates='user') + social_auth = relationship('UserAuthnzToken', back_populates='user') + stored_workflow_menu_entries = relationship('StoredWorkflowMenuEntry', + primaryjoin=(lambda: + (StoredWorkflowMenuEntry.user_id == User.id) # type: ignore + & (StoredWorkflowMenuEntry.stored_workflow_id == StoredWorkflow.id) # type: ignore + & not_(StoredWorkflow.deleted) # type: ignore + ), + backref='user', + cascade='all, delete-orphan', + collection_class=ordering_list('order_index')) + _preferences = relationship('UserPreference', + backref='user', + collection_class=attribute_mapped_collection('name')) + values = relationship('FormValues', + primaryjoin=(lambda: User.form_values_id == FormValues.id)) # type: ignore + # Add type hint (will this work w/SA?) + api_keys: 'List[APIKeys]' = relationship('APIKeys', + back_populates='user', + order_by='desc(APIKeys.create_time)') + pages = relationship('Page', back_populates='user') + reset_tokens = relationship('PasswordResetToken', back_populates='user') + histories_shared_by_others = relationship('HistoryUserShareAssociation', back_populates='user') + data_manager_histories = relationship('DataManagerHistoryAssociation', back_populates='user') + workflows_shared_by_others = relationship('StoredWorkflowUserShareAssociation', back_populates='user') + roles = relationship('UserRoleAssociation', back_populates='user') + stored_workflows = relationship('StoredWorkflow', back_populates='user', + primaryjoin=(lambda: User.id == StoredWorkflow.user_id)) # type: ignore + # attributes that will be accessed and returned when calling to_dict( view='collection' ) dict_collection_visible_keys = ['id', 'email', 'username', 'deleted', 'active', 'last_password_change'] # attributes that will be accessed and returned when calling to_dict( view='element' ) @@ -2858,9 +2926,9 @@ class UserRoleAssociation(Base, RepresentById): backref="non_private_roles", viewonly=True, primaryjoin=(lambda: - (User.table.c.id == UserRoleAssociation.user_id) # type: ignore + (User.id == UserRoleAssociation.user_id) # type: ignore & (UserRoleAssociation.role_id == Role.id) # type: ignore - & not_(Role.name == User.table.c.email)) # type: ignore + & not_(Role.name == User.email)) # type: ignore ) def __init__(self, user, role): @@ -6147,7 +6215,7 @@ class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): published = Column(Boolean, index=True, default=False) user = relationship('User', - primaryjoin=(lambda: User.table.c.id == StoredWorkflow.user_id), # type: ignore + primaryjoin=(lambda: User.id == StoredWorkflow.user_id), # type: ignore back_populates='stored_workflows') workflows = relationship('Workflow', back_populates='stored_workflow', @@ -7975,7 +8043,7 @@ class UserAuthnzToken(Base, UserMixin, RepresentById): @classmethod def get_users_by_email(cls, email): - return cls.user_query().filter(func.lower(User.table.c.email) == email.lower()) + return cls.user_query().filter(func.lower(User.email) == email.lower()) @classmethod def get_social_auth(cls, provider, uid): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index a79c6e8e150..9899c88174c 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -12,11 +12,9 @@ from sqlalchemy import ( Boolean, Column, DateTime, - desc, false, ForeignKey, Integer, - not_, Numeric, select, Table, @@ -24,9 +22,7 @@ from sqlalchemy import ( true, ) from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.ext.orderinglist import ordering_list from sqlalchemy.orm import class_mapper, column_property, deferred, object_session, relation -from sqlalchemy.orm.collections import attribute_mapped_collection from sqlalchemy.sql import exists from galaxy import model @@ -48,24 +44,6 @@ log = logging.getLogger(__name__) metadata = mapper_registry.metadata -model.User.table = Table( - "galaxy_user", metadata, - Column("id", Integer, primary_key=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now), - Column("email", TrimmedString(255), index=True, nullable=False), - Column("username", TrimmedString(255), index=True, unique=True), - Column("password", TrimmedString(255), nullable=False), - Column("last_password_change", DateTime, default=now), - Column("external", Boolean, default=False), - Column("form_values_id", Integer, ForeignKey("form_values.id"), index=True), - Column("deleted", Boolean, index=True, default=False), - Column("purged", Boolean, index=True, default=False), - Column("disk_usage", Numeric(15, 0), index=True), - # Column("person_metadata", JSONType), # TODO: add persistent, configurable metadata rep for workflow creator - Column("active", Boolean, index=True, default=True, nullable=False), - Column("activation_token", TrimmedString(64), nullable=True, index=True)) - model.HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata, Column("id", Integer, primary_key=True), @@ -235,57 +213,57 @@ simple_mapping(model.Dataset, # returns a list of users that history is shared with. model.History.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore -mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( - addresses=relation(model.UserAddress, - back_populates='user', - order_by=desc(model.UserAddress.update_time)), - cloudauthz=relation(model.CloudAuthz, back_populates='user'), - custos_auth=relation(model.CustosAuthnzToken, back_populates='user'), - default_permissions=relation(model.DefaultUserPermissions, back_populates='user'), - groups=relation(model.UserGroupAssociation, back_populates='user'), - histories=relation(model.History, - backref="user", - order_by=desc(model.History.update_time)), - active_histories=relation(model.History, - primaryjoin=( - (model.History.user_id == model.User.table.c.id) - & (not_(model.History.deleted)) - ), - viewonly=True, - order_by=desc(model.History.update_time)), - galaxy_sessions=relation(model.GalaxySession, - backref="user", - order_by=desc(model.GalaxySession.update_time)), - pages_shared_by_others=relation(model.PageUserShareAssociation, back_populates='user'), - quotas=relation(model.UserQuotaAssociation, back_populates='user'), - social_auth=relation(model.UserAuthnzToken, back_populates='user'), - stored_workflow_menu_entries=relation(model.StoredWorkflowMenuEntry, - primaryjoin=( - (model.StoredWorkflowMenuEntry.user_id == model.User.table.c.id) - & (model.StoredWorkflowMenuEntry.stored_workflow_id == model.StoredWorkflow.id) - & not_(model.StoredWorkflow.deleted) - ), - backref="user", - cascade="all, delete-orphan", - collection_class=ordering_list('order_index')), - _preferences=relation(model.UserPreference, - backref="user", - collection_class=attribute_mapped_collection('name')), - values=relation(model.FormValues, - primaryjoin=(model.User.table.c.form_values_id == model.FormValues.id)), - api_keys=relation(model.APIKeys, - back_populates="user", - order_by=desc(model.APIKeys.create_time)), - pages=relation(model.Page, back_populates='user'), - reset_tokens=relation(model.PasswordResetToken, back_populates='user'), - histories_shared_by_others=relation( - model.HistoryUserShareAssociation, back_populates='user'), - data_manager_histories=relation(model.DataManagerHistoryAssociation, back_populates='user'), - workflows_shared_by_others=relation(model.StoredWorkflowUserShareAssociation, back_populates='user'), - roles=relation(model.UserRoleAssociation, back_populates='user'), - stored_workflows=relation(model.StoredWorkflow, back_populates='user', - primaryjoin=(lambda: model.User.table.c.id == model.StoredWorkflow.user_id)), # type: ignore -)) +# mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( +# addresses=relation(model.UserAddress, +# back_populates='user', +# order_by=desc(model.UserAddress.update_time)), +# cloudauthz=relation(model.CloudAuthz, back_populates='user'), +# custos_auth=relation(model.CustosAuthnzToken, back_populates='user'), +# default_permissions=relation(model.DefaultUserPermissions, back_populates='user'), +# groups=relation(model.UserGroupAssociation, back_populates='user'), +# histories=relation(model.History, +# backref="user", +# order_by=desc(model.History.update_time)), +# active_histories=relation(model.History, +# primaryjoin=( +# (model.History.user_id == model.User.table.c.id) +# & (not_(model.History.deleted)) +# ), +# viewonly=True, +# order_by=desc(model.History.update_time)), +# galaxy_sessions=relation(model.GalaxySession, +# backref="user", +# order_by=desc(model.GalaxySession.update_time)), +# pages_shared_by_others=relation(model.PageUserShareAssociation, back_populates='user'), +# quotas=relation(model.UserQuotaAssociation, back_populates='user'), +# social_auth=relation(model.UserAuthnzToken, back_populates='user'), +# stored_workflow_menu_entries=relation(model.StoredWorkflowMenuEntry, +# primaryjoin=( +# (model.StoredWorkflowMenuEntry.user_id == model.User.table.c.id) +# & (model.StoredWorkflowMenuEntry.stored_workflow_id == model.StoredWorkflow.id) +# & not_(model.StoredWorkflow.deleted) +# ), +# backref="user", +# cascade="all, delete-orphan", +# collection_class=ordering_list('order_index')), +# _preferences=relation(model.UserPreference, +# backref="user", +# collection_class=attribute_mapped_collection('name')), +# values=relation(model.FormValues, +# primaryjoin=(model.User.table.c.form_values_id == model.FormValues.id)), +# api_keys=relation(model.APIKeys, +# back_populates="user", +# order_by=desc(model.APIKeys.create_time)), +# pages=relation(model.Page, back_populates='user'), +# reset_tokens=relation(model.PasswordResetToken, back_populates='user'), +# histories_shared_by_others=relation( +# model.HistoryUserShareAssociation, back_populates='user'), +# data_manager_histories=relation(model.DataManagerHistoryAssociation, back_populates='user'), +# workflows_shared_by_others=relation(model.StoredWorkflowUserShareAssociation, back_populates='user'), +# roles=relation(model.UserRoleAssociation, back_populates='user'), +# stored_workflows=relation(model.StoredWorkflow, back_populates='user', +# primaryjoin=(lambda: model.User.table.c.id == model.StoredWorkflow.user_id)), # type: ignore +# )) # Set up proxy so that this syntax is possible: # .preferences[pref_name] = pref_value diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index f34ce5b1634..a47fc4c4ff5 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -4733,8 +4733,8 @@ class TestTransferJob(BaseTest): class TestUser(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'user' + def test_table(self, cls_): + assert cls_.__tablename__ == 'galaxy_user' def test_columns(self, session, cls_, form_values): create_time = datetime.now() From 06e0ebf5b3bd892f01515a5c1362f8d694a7e829 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 14 Aug 2021 00:15:28 -0400 Subject: [PATCH 186/225] Do not wrap Galaxy model classes with SafeStringWrapper Models are mapped declaratively, so SQLAlchemy's DeclarativeMeta is their metaclass. By overriding attribute access, SafeStringWrapper (called in wrap_with_safe_string) inteferes with SQLAlchemy's instrumentation. And everything dies. --- lib/galaxy/tools/evaluation.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 0b542b8c323..9c8df5a795e 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -4,6 +4,7 @@ import os import shlex import tempfile +import sqlalchemy from galaxy import model from galaxy.files import ProvidesUserFileSourcesUserContext @@ -428,6 +429,14 @@ class ToolEvaluator: skip = ['chromInfo'] + list(self.tool.template_macro_params.keys()) if not self.tool or not self.tool.options or self.tool.options.sanitize: for key, value in list(param_dict.items()): + + # Do not wrap Galaxy model classes with SafeStringWrapper. + # Models are mapped declaratively, so SQLAlchemy's DeclarativeMeta is their metaclass. + # By overriding attribute access, SafeStringWrapper (called in wrap_with_safe_string) + # inteferes with SQLAlchemy's instrumentation. And everything dies. + if type(value.__class__) == sqlalchemy.orm.decl_api.DeclarativeMeta: + continue + if key not in skip: # Remove key so that new wrapped object will occupy key slot del param_dict[key] From dddde2c334441c9265e26bcb5974d1c7e521dbff Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 14 Aug 2021 12:34:16 -0400 Subject: [PATCH 187/225] Move most post-mapping statements to model Incrementally moving stuff from mapping.py --- lib/galaxy/model/__init__.py | 76 ++++++++++++++++++++++++----------- lib/galaxy/model/mapping.py | 77 ------------------------------------ 2 files changed, 52 insertions(+), 101 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 0ee6b646527..ee97a68e9e3 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -64,6 +64,7 @@ from sqlalchemy import ( ) from sqlalchemy.exc import OperationalError from sqlalchemy.ext import hybrid +from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.orderinglist import ordering_list from sqlalchemy.orm import ( aliased, @@ -8946,6 +8947,9 @@ def _prepare_metadata_for_serialization(id_encoder, serialization_options, metad return processed_metadata +# The following CleanupEvent* models could be defined as tables only; +# however making them models keeps things simple and consistent. + class CleanupEvent(Base): __tablename__ = 'cleanup_event' @@ -9035,38 +9039,21 @@ class CleanupEventImplicitlyConvertedDatasetAssociationAssociation(Base): icda_id = Column(Integer, ForeignKey('implicitly_converted_dataset_association.id'), index=True) -# Must be defined after PageRatingAssociation has been mapped -Page.average_rating = column_property( - select(func.avg(PageRatingAssociation.rating)).where(PageRatingAssociation.page_id == Page.id).scalar_subquery(), - deferred=True -) -# Must be defined after HistoryUserShareAssociation has been mapped -History.users_shared_with_count = column_property( - select(func.count(HistoryUserShareAssociation.id)).where(History.id == HistoryUserShareAssociation.history_id).scalar_subquery(), - deferred=True -) -# Must be defined after HistoryRatingAssociation has been mapped +# ---------------------------------------------------------------------------------------- +# The following statements must not precede the mapped models defined above. + History.average_rating = column_property( select(func.avg(HistoryRatingAssociation.rating)).where(HistoryRatingAssociation.history_id == History.id).scalar_subquery(), deferred=True ) -# Must be defined after VisualizationRatingAssociation has been mapped -Visualization.average_rating = column_property( - select(func.avg(VisualizationRatingAssociation.rating)).where(VisualizationRatingAssociation.visualization_id == Visualization.id).scalar_subquery(), +History.users_shared_with_count = column_property( + select(func.count(HistoryUserShareAssociation.id)).where(History.id == HistoryUserShareAssociation.history_id).scalar_subquery(), deferred=True ) -# Must be defined after WorkflowInvocationToSubworkflowInvocationAssociation -WorkflowInvocationStep.subworkflow_invocation_id = column_property( - select(WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id).where(and_( - WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id == WorkflowInvocationStep.workflow_invocation_id, - WorkflowInvocationToSubworkflowInvocationAssociation.workflow_step_id == WorkflowInvocationStep.workflow_step_id, - )).scalar_subquery(), -) - -Workflow.step_count = column_property( - select(func.count(WorkflowStep.id)).where(Workflow.id == WorkflowStep.workflow_id).scalar_subquery(), +Page.average_rating = column_property( + select(func.avg(PageRatingAssociation.rating)).where(PageRatingAssociation.page_id == Page.id).scalar_subquery(), deferred=True ) @@ -9074,3 +9061,44 @@ StoredWorkflow.average_rating = column_property( select(func.avg(StoredWorkflowRatingAssociation.rating)).where(StoredWorkflowRatingAssociation.stored_workflow_id == StoredWorkflow.id).scalar_subquery(), deferred=True ) + +Visualization.average_rating = column_property( + select(func.avg(VisualizationRatingAssociation.rating)).where(VisualizationRatingAssociation.visualization_id == Visualization.id).scalar_subquery(), + deferred=True +) + +Workflow.step_count = column_property( + select(func.count(WorkflowStep.id)).where(Workflow.id == WorkflowStep.workflow_id).scalar_subquery(), + deferred=True +) + +WorkflowInvocationStep.subworkflow_invocation_id = column_property( + select(WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id).where(and_( + WorkflowInvocationToSubworkflowInvocationAssociation.workflow_invocation_id == WorkflowInvocationStep.workflow_invocation_id, + WorkflowInvocationToSubworkflowInvocationAssociation.workflow_step_id == WorkflowInvocationStep.workflow_step_id, + )).scalar_subquery(), +) + +# Set up proxy so that +# History.users_shared_with +# returns a list of users that history is shared with. +History.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore + +# Set up proxy so that +# Page.users_shared_with +# returns a list of users that page is shared with. +Page.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore + +# Set up proxy so that +# StoredWorkflow.users_shared_with +# returns a list of users that workflow is shared with. +StoredWorkflow.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore + +# Set up proxy so that this syntax is possible: +# .preferences[pref_name] = pref_value +User.preferences = association_proxy('_preferences', 'value', creator=UserPreference) # type: ignore + +# Set up proxy so that +# Visualization.users_shared_with +# returns a list of users that visualization is shared with. +Visualization.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 9899c88174c..c6518372e40 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -21,7 +21,6 @@ from sqlalchemy import ( TEXT, true, ) -from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.orm import class_mapper, column_property, deferred, object_session, relation from sqlalchemy.sql import exists @@ -208,67 +207,6 @@ simple_mapping(model.Dataset, genome_index_tool_data=relation(model.GenomeIndexToolData, back_populates='dataset'), ) -# Set up proxy so that -# History.users_shared_with -# returns a list of users that history is shared with. -model.History.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore - -# mapper_registry.map_imperatively(model.User, model.User.table, properties=dict( -# addresses=relation(model.UserAddress, -# back_populates='user', -# order_by=desc(model.UserAddress.update_time)), -# cloudauthz=relation(model.CloudAuthz, back_populates='user'), -# custos_auth=relation(model.CustosAuthnzToken, back_populates='user'), -# default_permissions=relation(model.DefaultUserPermissions, back_populates='user'), -# groups=relation(model.UserGroupAssociation, back_populates='user'), -# histories=relation(model.History, -# backref="user", -# order_by=desc(model.History.update_time)), -# active_histories=relation(model.History, -# primaryjoin=( -# (model.History.user_id == model.User.table.c.id) -# & (not_(model.History.deleted)) -# ), -# viewonly=True, -# order_by=desc(model.History.update_time)), -# galaxy_sessions=relation(model.GalaxySession, -# backref="user", -# order_by=desc(model.GalaxySession.update_time)), -# pages_shared_by_others=relation(model.PageUserShareAssociation, back_populates='user'), -# quotas=relation(model.UserQuotaAssociation, back_populates='user'), -# social_auth=relation(model.UserAuthnzToken, back_populates='user'), -# stored_workflow_menu_entries=relation(model.StoredWorkflowMenuEntry, -# primaryjoin=( -# (model.StoredWorkflowMenuEntry.user_id == model.User.table.c.id) -# & (model.StoredWorkflowMenuEntry.stored_workflow_id == model.StoredWorkflow.id) -# & not_(model.StoredWorkflow.deleted) -# ), -# backref="user", -# cascade="all, delete-orphan", -# collection_class=ordering_list('order_index')), -# _preferences=relation(model.UserPreference, -# backref="user", -# collection_class=attribute_mapped_collection('name')), -# values=relation(model.FormValues, -# primaryjoin=(model.User.table.c.form_values_id == model.FormValues.id)), -# api_keys=relation(model.APIKeys, -# back_populates="user", -# order_by=desc(model.APIKeys.create_time)), -# pages=relation(model.Page, back_populates='user'), -# reset_tokens=relation(model.PasswordResetToken, back_populates='user'), -# histories_shared_by_others=relation( -# model.HistoryUserShareAssociation, back_populates='user'), -# data_manager_histories=relation(model.DataManagerHistoryAssociation, back_populates='user'), -# workflows_shared_by_others=relation(model.StoredWorkflowUserShareAssociation, back_populates='user'), -# roles=relation(model.UserRoleAssociation, back_populates='user'), -# stored_workflows=relation(model.StoredWorkflow, back_populates='user', -# primaryjoin=(lambda: model.User.table.c.id == model.StoredWorkflow.user_id)), # type: ignore -# )) - -# Set up proxy so that this syntax is possible: -# .preferences[pref_name] = pref_value -model.User.preferences = association_proxy('_preferences', 'value', creator=model.UserPreference) # type: ignore - mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssociation.table, properties=dict( dataset=relation(model.Dataset, primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.dataset_id == model.Dataset.table.c.id), @@ -320,21 +258,6 @@ mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.L # ), # ) -# Set up proxy so that -# StoredWorkflow.users_shared_with -# returns a list of users that workflow is shared with. -model.StoredWorkflow.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore - -# Set up proxy so that -# Page.users_shared_with -# returns a list of users that page is shared with. -model.Page.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore - -# Set up proxy so that -# Visualization.users_shared_with -# returns a list of users that visualization is shared with. -model.Visualization.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore - model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], and_(model.Job.id == model.JobToOutputDatasetAssociation.job_id, From cba395254242f5ab792b5e27ed9cecf06d0d9d74 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 14 Aug 2021 13:04:52 -0400 Subject: [PATCH 188/225] Move Dataset table definition to model --- lib/galaxy/model/__init__.py | 28 +++++++++++++++++++++++++--- lib/galaxy/model/mapping.py | 21 --------------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index ee97a68e9e3..becc48dbbb6 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -52,6 +52,7 @@ from sqlalchemy import ( PrimaryKeyConstraint, select, String, + Table, TEXT, Text, text, @@ -133,11 +134,8 @@ AUTO_PROPAGATED_TAGS = ["name"] if TYPE_CHECKING: - from sqlalchemy.schema import Table - class _HasTable: table: Table - else: _HasTable = object @@ -9039,6 +9037,30 @@ class CleanupEventImplicitlyConvertedDatasetAssociationAssociation(Base): icda_id = Column(Integer, ForeignKey('implicitly_converted_dataset_association.id'), index=True) +# The following models (HDA, LDDA, Dataset) are mapped imperatively (for details see discussion in PR #12064) +# TLDR: there are issues ('metadata' property, Galaxy object wrapping) that need to be addressed separately +# before these models can be mapped declaratively. Keeping them in the mapping module breaks the auth package +# tests (which import model directly bypassing the mapping module); fixing that is possible by importing +# mapping into the test; however, having all models mapped in the same module is cleaner. + +Dataset.table = Table( + 'dataset', mapper_registry.metadata, + Column('id', Integer, primary_key=True), + Column('job_id', Integer, ForeignKey('job.id'), index=True, nullable=True), + Column('create_time', DateTime, default=now), + Column('update_time', DateTime, index=True, default=now, onupdate=now), + Column('state', TrimmedString(64), index=True), + Column('deleted', Boolean, index=True, default=False), + Column('purged', Boolean, index=True, default=False), + Column('purgable', Boolean, default=True), + Column('object_store_id', TrimmedString(255), index=True), + Column('external_filename', TEXT), + Column('_extra_files_path', TEXT), + Column('created_from_basename', TEXT), + Column('file_size', Numeric(15, 0)), + Column('total_size', Numeric(15, 0)), + Column('uuid', UUIDType())) + # ---------------------------------------------------------------------------------------- # The following statements must not precede the mapped models defined above. diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index c6518372e40..d8b86c469c1 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -15,7 +15,6 @@ from sqlalchemy import ( false, ForeignKey, Integer, - Numeric, select, Table, TEXT, @@ -30,7 +29,6 @@ from galaxy.model.base import SharedModelMapping from galaxy.model.custom_types import ( JSONType, TrimmedString, - UUIDType, ) from galaxy.model.migrate.triggers.update_audit_table import install as install_timestamp_triggers from galaxy.model.orm.engine_factory import build_engine @@ -75,25 +73,6 @@ model.HistoryDatasetAssociation.table = Table( Column("hidden_beneath_collection_instance_id", ForeignKey("history_dataset_collection_association.id"), nullable=True)) - -model.Dataset.table = Table( - "dataset", metadata, - Column("id", Integer, primary_key=True), - Column('job_id', Integer, ForeignKey('job.id'), index=True, nullable=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, index=True, default=now, onupdate=now), - Column("state", TrimmedString(64), index=True), - Column("deleted", Boolean, index=True, default=False), - Column("purged", Boolean, index=True, default=False), - Column("purgable", Boolean, default=True), - Column("object_store_id", TrimmedString(255), index=True), - Column("external_filename", TEXT), - Column("_extra_files_path", TEXT), - Column("created_from_basename", TEXT), - Column('file_size', Numeric(15, 0)), - Column('total_size', Numeric(15, 0)), - Column('uuid', UUIDType())) - model.LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_association", metadata, Column("id", Integer, primary_key=True), From 02e168fe66bca8ea948d3f5f97f22aff6ec82e5a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 14 Aug 2021 13:09:12 -0400 Subject: [PATCH 189/225] Move HDA table definition to model --- lib/galaxy/model/__init__.py | 32 ++++++++++++++++++++++++++++++++ lib/galaxy/model/mapping.py | 33 --------------------------------- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index becc48dbbb6..1c3742d69d3 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -9061,6 +9061,38 @@ Dataset.table = Table( Column('total_size', Numeric(15, 0)), Column('uuid', UUIDType())) +HistoryDatasetAssociation.table = Table( + 'history_dataset_association', mapper_registry.metadata, + Column('id', Integer, primary_key=True), + Column('history_id', Integer, ForeignKey('history.id'), index=True), + Column('dataset_id', Integer, ForeignKey('dataset.id'), index=True), + Column('create_time', DateTime, default=now), + Column('update_time', DateTime, default=now, onupdate=now, index=True), + Column('state', TrimmedString(64), index=True, key='_state'), + Column('copied_from_history_dataset_association_id', Integer, + ForeignKey('history_dataset_association.id'), nullable=True), + Column('copied_from_library_dataset_dataset_association_id', Integer, + ForeignKey('library_dataset_dataset_association.id'), nullable=True), + Column('name', TrimmedString(255)), + Column('info', TrimmedString(255)), + Column('blurb', TrimmedString(255)), + Column('peek', TEXT, key='_peek'), + Column('tool_version', TEXT), + Column('extension', TrimmedString(64)), + Column('metadata', JSONType, key='_metadata'), + Column('parent_id', Integer, ForeignKey('history_dataset_association.id'), nullable=True), + Column('designation', TrimmedString(255)), + Column('deleted', Boolean, index=True, default=False), + Column('visible', Boolean), + Column('extended_metadata_id', Integer, ForeignKey('extended_metadata.id'), index=True), + Column('version', Integer, default=1, nullable=True, index=True), + Column('hid', Integer), + Column('purged', Boolean, index=True, default=False), + Column('validated_state', TrimmedString(64), default='unvalidated', nullable=False), + Column('validated_state_message', TEXT), + Column('hidden_beneath_collection_instance_id', + ForeignKey('history_dataset_collection_association.id'), nullable=True)) + # ---------------------------------------------------------------------------------------- # The following statements must not precede the mapped models defined above. diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index d8b86c469c1..6b8ffd11cf3 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -40,39 +40,6 @@ log = logging.getLogger(__name__) metadata = mapper_registry.metadata - -model.HistoryDatasetAssociation.table = Table( - "history_dataset_association", metadata, - Column("id", Integer, primary_key=True), - Column("history_id", Integer, ForeignKey("history.id"), index=True), - Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now, index=True), - Column("state", TrimmedString(64), index=True, key="_state"), - Column("copied_from_history_dataset_association_id", Integer, - ForeignKey("history_dataset_association.id"), nullable=True), - Column("copied_from_library_dataset_dataset_association_id", Integer, - ForeignKey("library_dataset_dataset_association.id"), nullable=True), - Column("name", TrimmedString(255)), - Column("info", TrimmedString(255)), - Column("blurb", TrimmedString(255)), - Column("peek", TEXT, key="_peek"), - Column("tool_version", TEXT), - Column("extension", TrimmedString(64)), - Column("metadata", JSONType, key="_metadata"), - Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True), - Column("designation", TrimmedString(255)), - Column("deleted", Boolean, index=True, default=False), - Column("visible", Boolean), - Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True), - Column("version", Integer, default=1, nullable=True, index=True), - Column("hid", Integer), - Column("purged", Boolean, index=True, default=False), - Column("validated_state", TrimmedString(64), default='unvalidated', nullable=False), - Column("validated_state_message", TEXT), - Column("hidden_beneath_collection_instance_id", - ForeignKey("history_dataset_collection_association.id"), nullable=True)) - model.LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_association", metadata, Column("id", Integer, primary_key=True), From a1e6c03f3ae0508edab14d2552609e1dc5cc67e3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 14 Aug 2021 13:13:38 -0400 Subject: [PATCH 190/225] Move LDDA table definition to model --- lib/galaxy/model/__init__.py | 33 +++++++++++++++++++++++++++ lib/galaxy/model/mapping.py | 44 ------------------------------------ 2 files changed, 33 insertions(+), 44 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1c3742d69d3..5b3bcdbb82a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -9093,6 +9093,39 @@ HistoryDatasetAssociation.table = Table( Column('hidden_beneath_collection_instance_id', ForeignKey('history_dataset_collection_association.id'), nullable=True)) +LibraryDatasetDatasetAssociation.table = Table( + 'library_dataset_dataset_association', mapper_registry.metadata, + Column('id', Integer, primary_key=True), + Column('library_dataset_id', Integer, ForeignKey('library_dataset.id'), index=True), + Column('dataset_id', Integer, ForeignKey('dataset.id'), index=True), + Column('create_time', DateTime, default=now), + Column('update_time', DateTime, default=now, onupdate=now, index=True), + Column('state', TrimmedString(64), index=True, key='_state'), + Column('copied_from_history_dataset_association_id', Integer, + ForeignKey('history_dataset_association.id', + use_alter=True, name='history_dataset_association_dataset_id_fkey'), + nullable=True), + Column('copied_from_library_dataset_dataset_association_id', Integer, + ForeignKey('library_dataset_dataset_association.id', + use_alter=True, name='library_dataset_dataset_association_id_fkey'), + nullable=True), + Column('name', TrimmedString(255), index=True), + Column('info', TrimmedString(255)), + Column('blurb', TrimmedString(255)), + Column('peek', TEXT, key='_peek'), + Column('tool_version', TEXT), + Column('extension', TrimmedString(64)), + Column('metadata', JSONType, key='_metadata'), + Column('parent_id', Integer, ForeignKey('library_dataset_dataset_association.id'), nullable=True), + Column('designation', TrimmedString(255)), + Column('deleted', Boolean, index=True, default=False), + Column('validated_state', TrimmedString(64), default='unvalidated', nullable=False), + Column('validated_state_message', TEXT), + Column('visible', Boolean), + Column('extended_metadata_id', Integer, ForeignKey('extended_metadata.id'), index=True), + Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True), + Column('message', TrimmedString(255))) + # ---------------------------------------------------------------------------------------- # The following statements must not precede the mapped models defined above. diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 6b8ffd11cf3..811f6c39ab7 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -9,15 +9,8 @@ from typing import Optional, Type from sqlalchemy import ( and_, - Boolean, - Column, - DateTime, false, - ForeignKey, - Integer, select, - Table, - TEXT, true, ) from sqlalchemy.orm import class_mapper, column_property, deferred, object_session, relation @@ -26,10 +19,6 @@ from sqlalchemy.sql import exists from galaxy import model from galaxy.model import mapper_registry from galaxy.model.base import SharedModelMapping -from galaxy.model.custom_types import ( - JSONType, - TrimmedString, -) from galaxy.model.migrate.triggers.update_audit_table import install as install_timestamp_triggers from galaxy.model.orm.engine_factory import build_engine from galaxy.model.orm.now import now @@ -40,39 +29,6 @@ log = logging.getLogger(__name__) metadata = mapper_registry.metadata -model.LibraryDatasetDatasetAssociation.table = Table( - "library_dataset_dataset_association", metadata, - Column("id", Integer, primary_key=True), - Column("library_dataset_id", Integer, ForeignKey("library_dataset.id"), index=True), - Column("dataset_id", Integer, ForeignKey("dataset.id"), index=True), - Column("create_time", DateTime, default=now), - Column("update_time", DateTime, default=now, onupdate=now, index=True), - Column("state", TrimmedString(64), index=True, key="_state"), - Column("copied_from_history_dataset_association_id", Integer, - ForeignKey("history_dataset_association.id", use_alter=True, name='history_dataset_association_dataset_id_fkey'), - nullable=True), - - Column("copied_from_library_dataset_dataset_association_id", Integer, - ForeignKey("library_dataset_dataset_association.id", use_alter=True, name='library_dataset_dataset_association_id_fkey'), - nullable=True), - - Column("name", TrimmedString(255), index=True), - Column("info", TrimmedString(255)), - Column("blurb", TrimmedString(255)), - Column("peek", TEXT, key="_peek"), - Column("tool_version", TEXT), - Column("extension", TrimmedString(64)), - Column("metadata", JSONType, key="_metadata"), - Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True), - Column("designation", TrimmedString(255)), - Column("deleted", Boolean, index=True, default=False), - Column("validated_state", TrimmedString(64), default='unvalidated', nullable=False), - Column("validated_state_message", TEXT), - Column("visible", Boolean), - Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True), - Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True), - Column("message", TrimmedString(255))) - # With the tables defined we can define the mappers and setup the # relationships between the model objects. From f9ae69cf89369158fc4d08961612f329b6dc8846 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 14 Aug 2021 13:17:54 -0400 Subject: [PATCH 191/225] Move Dataset mapping statement to model --- lib/galaxy/model/__init__.py | 32 +++++++++++++++++++++++++++++++- lib/galaxy/model/mapping.py | 26 -------------------------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 5b3bcdbb82a..3b4b1f8dc7a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -40,6 +40,7 @@ from sqlalchemy import ( Column, DateTime, desc, + false, ForeignKey, func, Index, @@ -9037,7 +9038,7 @@ class CleanupEventImplicitlyConvertedDatasetAssociationAssociation(Base): icda_id = Column(Integer, ForeignKey('implicitly_converted_dataset_association.id'), index=True) -# The following models (HDA, LDDA, Dataset) are mapped imperatively (for details see discussion in PR #12064) +# The following models (Dataset, HDA, LDDA) are mapped imperatively (for details see discussion in PR #12064) # TLDR: there are issues ('metadata' property, Galaxy object wrapping) that need to be addressed separately # before these models can be mapped declaratively. Keeping them in the mapping module breaks the auth package # tests (which import model directly bypassing the mapping module); fixing that is possible by importing @@ -9126,6 +9127,35 @@ LibraryDatasetDatasetAssociation.table = Table( Column('user_id', Integer, ForeignKey('galaxy_user.id'), index=True), Column('message', TrimmedString(255))) +mapper_registry.map_imperatively( + Dataset, + Dataset.table, + properties=dict( + actions=relationship(DatasetPermissions, back_populates='dataset'), + job=relationship(Job, primaryjoin=(Dataset.table.c.job_id == Job.id)), + active_history_associations=relationship(HistoryDatasetAssociation, + primaryjoin=( + (Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id) + & (HistoryDatasetAssociation.table.c.deleted == false()) + & (HistoryDatasetAssociation.table.c.purged == false())), + viewonly=True), + purged_history_associations=relationship(HistoryDatasetAssociation, + primaryjoin=( + (Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id) + & (HistoryDatasetAssociation.table.c.purged == true())), + viewonly=True), + active_library_associations=relationship(LibraryDatasetDatasetAssociation, + primaryjoin=( + (Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id) + & (LibraryDatasetDatasetAssociation.table.c.deleted == false())), + viewonly=True), + hashes=relationship(DatasetHash, back_populates='dataset'), + sources=relationship(DatasetSource, back_populates='dataset'), + job_export_history_archive=relationship(JobExportHistoryArchive, back_populates='dataset'), + genome_index_tool_data=relationship(GenomeIndexToolData, back_populates='dataset'), + ) +) + # ---------------------------------------------------------------------------------------- # The following statements must not precede the mapped models defined above. diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 811f6c39ab7..efde212345b 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -9,7 +9,6 @@ from typing import Optional, Type from sqlalchemy import ( and_, - false, select, true, ) @@ -84,31 +83,6 @@ simple_mapping(model.HistoryDatasetAssociation, back_populates='dataset') ) -simple_mapping(model.Dataset, - actions=relation(model.DatasetPermissions, back_populates='dataset'), - job=relation(model.Job, primaryjoin=(model.Dataset.table.c.job_id == model.Job.id)), - active_history_associations=relation(model.HistoryDatasetAssociation, - primaryjoin=( - (model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id) - & (model.HistoryDatasetAssociation.table.c.deleted == false()) - & (model.HistoryDatasetAssociation.table.c.purged == false())), - viewonly=True), - purged_history_associations=relation(model.HistoryDatasetAssociation, - primaryjoin=( - (model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id) - & (model.HistoryDatasetAssociation.table.c.purged == true())), - viewonly=True), - active_library_associations=relation(model.LibraryDatasetDatasetAssociation, - primaryjoin=( - (model.Dataset.table.c.id == model.LibraryDatasetDatasetAssociation.table.c.dataset_id) - & (model.LibraryDatasetDatasetAssociation.table.c.deleted == false())), - viewonly=True), - hashes=relation(model.DatasetHash, back_populates='dataset'), - sources=relation(model.DatasetSource, back_populates='dataset'), - job_export_history_archive=relation(model.JobExportHistoryArchive, back_populates='dataset'), - genome_index_tool_data=relation(model.GenomeIndexToolData, back_populates='dataset'), -) - mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssociation.table, properties=dict( dataset=relation(model.Dataset, primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.dataset_id == model.Dataset.table.c.id), From 350b07353e38382953935d8ebad0b62992ffc93a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 14 Aug 2021 13:23:44 -0400 Subject: [PATCH 192/225] Move HDA mapping statement to model --- lib/galaxy/model/__init__.py | 53 ++++++++++++++++++++++++++++++++++++ lib/galaxy/model/mapping.py | 48 -------------------------------- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3b4b1f8dc7a..23859db45f6 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -71,6 +71,7 @@ from sqlalchemy.ext.orderinglist import ordering_list from sqlalchemy.orm import ( aliased, column_property, + deferred, joinedload, object_session, Query, @@ -9156,6 +9157,58 @@ mapper_registry.map_imperatively( ) ) +mapper_registry.map_imperatively( + HistoryDatasetAssociation, + HistoryDatasetAssociation.table, + properties=dict( + dataset=relationship(Dataset, + primaryjoin=(Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id), + lazy=False, + backref='history_associations'), + copied_from_history_dataset_association=relationship(HistoryDatasetAssociation, + primaryjoin=(HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id + == HistoryDatasetAssociation.table.c.id), + remote_side=[HistoryDatasetAssociation.table.c.id], + uselist=False, + backref='copied_to_history_dataset_associations'), + copied_to_library_dataset_dataset_associations=relationship(LibraryDatasetDatasetAssociation, + primaryjoin=(HistoryDatasetAssociation.table.c.id + == LibraryDatasetDatasetAssociation.table.c.copied_from_history_dataset_association_id), + backref='copied_from_history_dataset_association'), + tags=relationship(HistoryDatasetAssociationTagAssociation, + order_by=HistoryDatasetAssociationTagAssociation.id, + back_populates='history_dataset_association'), + annotations=relationship(HistoryDatasetAssociationAnnotationAssociation, + order_by=HistoryDatasetAssociationAnnotationAssociation.id, + back_populates="hda"), + ratings=relationship(HistoryDatasetAssociationRatingAssociation, + order_by=HistoryDatasetAssociationRatingAssociation.id, + back_populates="history_dataset_association"), + extended_metadata=relationship(ExtendedMetadata, + primaryjoin=(HistoryDatasetAssociation.table.c.extended_metadata_id + == ExtendedMetadata.id)), + hidden_beneath_collection_instance=relationship(HistoryDatasetCollectionAssociation, + primaryjoin=(HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id + == HistoryDatasetCollectionAssociation.id), + uselist=False, + backref="hidden_dataset_instances"), + _metadata=deferred(HistoryDatasetAssociation.table.c._metadata), + dependent_jobs=relationship(JobToInputDatasetAssociation, back_populates='dataset'), + creating_job_associations=relationship( + JobToOutputDatasetAssociation, back_populates='dataset'), + history=relationship(History, back_populates='datasets'), + implicitly_converted_datasets=relationship(ImplicitlyConvertedDatasetAssociation, + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_parent_id # type: ignore + == HistoryDatasetAssociation.id), # type: ignore + back_populates='parent_hda'), + implicitly_converted_parent_datasets=relationship(ImplicitlyConvertedDatasetAssociation, + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_id # type: ignore + == HistoryDatasetAssociation.id), # type: ignore + back_populates='dataset') + ) +) + + # ---------------------------------------------------------------------------------------- # The following statements must not precede the mapped models defined above. diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index efde212345b..0c9a28395fa 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -35,54 +35,6 @@ def simple_mapping(model, **kwds): mapper_registry.map_imperatively(model, model.table, properties=kwds) -simple_mapping(model.HistoryDatasetAssociation, - dataset=relation(model.Dataset, - primaryjoin=(model.Dataset.table.c.id == model.HistoryDatasetAssociation.table.c.dataset_id), - lazy=False, - backref='history_associations'), - # .history defined in History mapper - copied_from_history_dataset_association=relation(model.HistoryDatasetAssociation, - primaryjoin=(model.HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id - == model.HistoryDatasetAssociation.table.c.id), - remote_side=[model.HistoryDatasetAssociation.table.c.id], - uselist=False, - backref='copied_to_history_dataset_associations'), - copied_to_library_dataset_dataset_associations=relation(model.LibraryDatasetDatasetAssociation, - primaryjoin=(model.HistoryDatasetAssociation.table.c.id - == model.LibraryDatasetDatasetAssociation.table.c.copied_from_history_dataset_association_id), - backref='copied_from_history_dataset_association'), - tags=relation(model.HistoryDatasetAssociationTagAssociation, - order_by=model.HistoryDatasetAssociationTagAssociation.id, - back_populates='history_dataset_association'), - annotations=relation(model.HistoryDatasetAssociationAnnotationAssociation, - order_by=model.HistoryDatasetAssociationAnnotationAssociation.id, - back_populates="hda"), - ratings=relation(model.HistoryDatasetAssociationRatingAssociation, - order_by=model.HistoryDatasetAssociationRatingAssociation.id, - back_populates="history_dataset_association"), - extended_metadata=relation(model.ExtendedMetadata, - primaryjoin=(model.HistoryDatasetAssociation.table.c.extended_metadata_id - == model.ExtendedMetadata.id)), - hidden_beneath_collection_instance=relation(model.HistoryDatasetCollectionAssociation, - primaryjoin=(model.HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id - == model.HistoryDatasetCollectionAssociation.id), - uselist=False, - backref="hidden_dataset_instances"), - _metadata=deferred(model.HistoryDatasetAssociation.table.c._metadata), - dependent_jobs=relation(model.JobToInputDatasetAssociation, back_populates='dataset'), - creating_job_associations=relation( - model.JobToOutputDatasetAssociation, back_populates='dataset'), - history=relation(model.History, back_populates='datasets'), - implicitly_converted_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, - primaryjoin=(lambda: model.ImplicitlyConvertedDatasetAssociation.hda_parent_id # type: ignore - == model.HistoryDatasetAssociation.id), # type: ignore - back_populates='parent_hda'), - implicitly_converted_parent_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, - primaryjoin=(lambda: model.ImplicitlyConvertedDatasetAssociation.hda_id # type: ignore - == model.HistoryDatasetAssociation.id), # type: ignore - back_populates='dataset') -) - mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssociation.table, properties=dict( dataset=relation(model.Dataset, primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.dataset_id == model.Dataset.table.c.id), From 3be3f16a39cb5d571481b7b12e24a3741678eb0f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 14 Aug 2021 13:28:55 -0400 Subject: [PATCH 193/225] Move LDDA mapping statement to model --- lib/galaxy/model/__init__.py | 46 +++++++++++++++++++++++++++ lib/galaxy/model/mapping.py | 60 +----------------------------------- 2 files changed, 47 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 23859db45f6..55aabc817ea 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -9208,6 +9208,52 @@ mapper_registry.map_imperatively( ) ) +mapper_registry.map_imperatively( + LibraryDatasetDatasetAssociation, + LibraryDatasetDatasetAssociation.table, + properties=dict( + dataset=relationship(Dataset, + primaryjoin=(LibraryDatasetDatasetAssociation.table.c.dataset_id == Dataset.table.c.id), + backref='library_associations'), + library_dataset=relationship(LibraryDataset, + foreign_keys=LibraryDatasetDatasetAssociation.table.c.library_dataset_id), + user=relationship(User), + copied_from_library_dataset_dataset_association=relationship(LibraryDatasetDatasetAssociation, + primaryjoin=( + LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id + == LibraryDatasetDatasetAssociation.table.c.id), + remote_side=[LibraryDatasetDatasetAssociation.table.c.id], + uselist=False, + backref='copied_to_library_dataset_dataset_associations'), + copied_to_history_dataset_associations=relationship(HistoryDatasetAssociation, + primaryjoin=(LibraryDatasetDatasetAssociation.table.c.id + == HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id), + backref='copied_from_library_dataset_dataset_association'), + implicitly_converted_datasets=relationship(ImplicitlyConvertedDatasetAssociation, + primaryjoin=(ImplicitlyConvertedDatasetAssociation.ldda_parent_id + == LibraryDatasetDatasetAssociation.table.c.id), + backref='parent_ldda'), + tags=relationship(LibraryDatasetDatasetAssociationTagAssociation, + order_by=LibraryDatasetDatasetAssociationTagAssociation.id, + back_populates='library_dataset_dataset_association'), + extended_metadata=relationship(ExtendedMetadata, + primaryjoin=(LibraryDatasetDatasetAssociation.table.c.extended_metadata_id + == ExtendedMetadata.id) + ), + _metadata=deferred(LibraryDatasetDatasetAssociation.table.c._metadata), + actions=relationship( + LibraryDatasetDatasetAssociationPermissions, + back_populates='library_dataset_dataset_association'), + dependent_jobs=relationship( + JobToInputLibraryDatasetAssociation, back_populates='dataset'), + creating_job_associations=relationship( + JobToOutputLibraryDatasetAssociation, back_populates='dataset'), + implicitly_converted_parent_datasets=relationship(ImplicitlyConvertedDatasetAssociation, + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_id # type: ignore + == LibraryDatasetDatasetAssociation.id), # type: ignore + back_populates='dataset_ldda'), + ) +) # ---------------------------------------------------------------------------------------- # The following statements must not precede the mapped models defined above. diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 0c9a28395fa..283377f6544 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -12,7 +12,7 @@ from sqlalchemy import ( select, true, ) -from sqlalchemy.orm import class_mapper, column_property, deferred, object_session, relation +from sqlalchemy.orm import class_mapper, column_property, object_session, relation from sqlalchemy.sql import exists from galaxy import model @@ -28,64 +28,6 @@ log = logging.getLogger(__name__) metadata = mapper_registry.metadata - -# With the tables defined we can define the mappers and setup the -# relationships between the model objects. -def simple_mapping(model, **kwds): - mapper_registry.map_imperatively(model, model.table, properties=kwds) - - -mapper_registry.map_imperatively(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssociation.table, properties=dict( - dataset=relation(model.Dataset, - primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.dataset_id == model.Dataset.table.c.id), - backref='library_associations'), - library_dataset=relation(model.LibraryDataset, - foreign_keys=model.LibraryDatasetDatasetAssociation.table.c.library_dataset_id), - # user=relation( model.User.mapper ), - user=relation(model.User), - copied_from_library_dataset_dataset_association=relation(model.LibraryDatasetDatasetAssociation, - primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id - == model.LibraryDatasetDatasetAssociation.table.c.id), - remote_side=[model.LibraryDatasetDatasetAssociation.table.c.id], - uselist=False, - backref='copied_to_library_dataset_dataset_associations'), - copied_to_history_dataset_associations=relation(model.HistoryDatasetAssociation, - primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.id - == model.HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id), - backref='copied_from_library_dataset_dataset_association'), - implicitly_converted_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, - primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.ldda_parent_id - == model.LibraryDatasetDatasetAssociation.table.c.id), - backref='parent_ldda'), - tags=relation(model.LibraryDatasetDatasetAssociationTagAssociation, - order_by=model.LibraryDatasetDatasetAssociationTagAssociation.id, - back_populates='library_dataset_dataset_association'), - extended_metadata=relation(model.ExtendedMetadata, - primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.extended_metadata_id == model.ExtendedMetadata.id) - ), - _metadata=deferred(model.LibraryDatasetDatasetAssociation.table.c._metadata), - actions=relation( - model.LibraryDatasetDatasetAssociationPermissions, - back_populates='library_dataset_dataset_association'), - dependent_jobs=relation( - model.JobToInputLibraryDatasetAssociation, back_populates='dataset'), - creating_job_associations=relation( - model.JobToOutputLibraryDatasetAssociation, back_populates='dataset'), - implicitly_converted_parent_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, - primaryjoin=(lambda: model.ImplicitlyConvertedDatasetAssociation.ldda_id # type: ignore - == model.LibraryDatasetDatasetAssociation.id), # type: ignore - back_populates='dataset_ldda'), -)) - -# simple_mapping( -# model.ImplicitCollectionJobsHistoryDatasetCollectionAssociation, -# history_dataset_collection_associations=relation( -# model.HistoryDatasetCollectionAssociation, -# backref=backref("implicit_collection_jobs_association", uselist=False), -# uselist=True, -# ), -# ) - model.Job.any_output_dataset_deleted = column_property( # type: ignore exists([model.HistoryDatasetAssociation], and_(model.Job.id == model.JobToOutputDatasetAssociation.job_id, From 19eb876eb98c6cf4b653baf4e4337a02159d7fa5 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 14 Aug 2021 13:40:11 -0400 Subject: [PATCH 194/225] Add remaining column properties to model --- lib/galaxy/model/__init__.py | 17 +++++++++++++++++ lib/galaxy/model/mapping.py | 19 +------------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 55aabc817ea..b7c7176c670 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -81,6 +81,7 @@ from sqlalchemy.orm import ( ) from sqlalchemy.orm.collections import attribute_mapped_collection from sqlalchemy.orm.decl_api import DeclarativeMeta +from sqlalchemy.sql import exists import galaxy.exceptions import galaxy.model.metadata @@ -9258,6 +9259,22 @@ mapper_registry.map_imperatively( # ---------------------------------------------------------------------------------------- # The following statements must not precede the mapped models defined above. +Job.any_output_dataset_collection_instances_deleted = column_property( # type: ignore + exists([HistoryDatasetCollectionAssociation.id], and_( + Job.id == JobToOutputDatasetCollectionAssociation.job_id, + HistoryDatasetCollectionAssociation.id == JobToOutputDatasetCollectionAssociation.dataset_collection_id, + HistoryDatasetCollectionAssociation.deleted == true()) + ) +) + +Job.any_output_dataset_deleted = column_property( # type: ignore + exists([HistoryDatasetAssociation], and_( + Job.id == JobToOutputDatasetAssociation.job_id, + HistoryDatasetAssociation.table.c.id == JobToOutputDatasetAssociation.dataset_id, + HistoryDatasetAssociation.table.c.deleted == true()) + ) +) + History.average_rating = column_property( select(func.avg(HistoryRatingAssociation.rating)).where(HistoryRatingAssociation.history_id == History.id).scalar_subquery(), deferred=True diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 283377f6544..ce5ef43327d 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -10,10 +10,8 @@ from typing import Optional, Type from sqlalchemy import ( and_, select, - true, ) -from sqlalchemy.orm import class_mapper, column_property, object_session, relation -from sqlalchemy.sql import exists +from sqlalchemy.orm import class_mapper, object_session, relation from galaxy import model from galaxy.model import mapper_registry @@ -28,21 +26,6 @@ log = logging.getLogger(__name__) metadata = mapper_registry.metadata -model.Job.any_output_dataset_deleted = column_property( # type: ignore - exists([model.HistoryDatasetAssociation], - and_(model.Job.id == model.JobToOutputDatasetAssociation.job_id, - model.HistoryDatasetAssociation.table.c.id == model.JobToOutputDatasetAssociation.dataset_id, - model.HistoryDatasetAssociation.table.c.deleted == true()) - ) -) -model.Job.any_output_dataset_collection_instances_deleted = column_property( # type: ignore - exists([model.HistoryDatasetCollectionAssociation.id], - and_(model.Job.id == model.JobToOutputDatasetCollectionAssociation.job_id, - model.HistoryDatasetCollectionAssociation.id == model.JobToOutputDatasetCollectionAssociation.dataset_collection_id, - model.HistoryDatasetCollectionAssociation.deleted == true()) - ) -) - class_mapper(model.HistoryDatasetCollectionAssociation).add_property( "creating_job_associations", relation(model.JobToOutputDatasetCollectionAssociation, viewonly=True)) From f2673538517f4e16afda7fe9e6e146ee771a569b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 12:05:55 -0400 Subject: [PATCH 195/225] Cleanup: remove unnecessary type:ignore statements Also, declare column_property attributes inline --- lib/galaxy/model/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index b7c7176c670..dd24639ebcf 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -943,6 +943,9 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): workflow_invocation_step = relationship('WorkflowInvocationStep', back_populates='job', uselist=False) + any_output_dataset_collection_instances_deleted: column_property # defined at the end of this module + any_output_dataset_deleted: column_property # defined at the end of this module + dict_collection_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version'] dict_element_visible_keys = ['id', 'state', 'exit_code', 'update_time', 'create_time', 'galaxy_version', 'command_version'] @@ -9199,12 +9202,12 @@ mapper_registry.map_imperatively( JobToOutputDatasetAssociation, back_populates='dataset'), history=relationship(History, back_populates='datasets'), implicitly_converted_datasets=relationship(ImplicitlyConvertedDatasetAssociation, - primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_parent_id # type: ignore - == HistoryDatasetAssociation.id), # type: ignore + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_parent_id + == HistoryDatasetAssociation.id), back_populates='parent_hda'), implicitly_converted_parent_datasets=relationship(ImplicitlyConvertedDatasetAssociation, - primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_id # type: ignore - == HistoryDatasetAssociation.id), # type: ignore + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_id + == HistoryDatasetAssociation.id), back_populates='dataset') ) ) @@ -9250,8 +9253,8 @@ mapper_registry.map_imperatively( creating_job_associations=relationship( JobToOutputLibraryDatasetAssociation, back_populates='dataset'), implicitly_converted_parent_datasets=relationship(ImplicitlyConvertedDatasetAssociation, - primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_id # type: ignore - == LibraryDatasetDatasetAssociation.id), # type: ignore + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_id + == LibraryDatasetDatasetAssociation.id), back_populates='dataset_ldda'), ) ) @@ -9259,7 +9262,7 @@ mapper_registry.map_imperatively( # ---------------------------------------------------------------------------------------- # The following statements must not precede the mapped models defined above. -Job.any_output_dataset_collection_instances_deleted = column_property( # type: ignore +Job.any_output_dataset_collection_instances_deleted = column_property( exists([HistoryDatasetCollectionAssociation.id], and_( Job.id == JobToOutputDatasetCollectionAssociation.job_id, HistoryDatasetCollectionAssociation.id == JobToOutputDatasetCollectionAssociation.dataset_collection_id, @@ -9267,7 +9270,7 @@ Job.any_output_dataset_collection_instances_deleted = column_property( # type: ) ) -Job.any_output_dataset_deleted = column_property( # type: ignore +Job.any_output_dataset_deleted = column_property( exists([HistoryDatasetAssociation], and_( Job.id == JobToOutputDatasetAssociation.job_id, HistoryDatasetAssociation.table.c.id == JobToOutputDatasetAssociation.dataset_id, From c7a7e68bb998d9c30ec4a99a2a34407ad1a00c1f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 12:21:08 -0400 Subject: [PATCH 196/225] Cleanup: declare association proxies inline Make comments consistent --- lib/galaxy/model/__init__.py | 60 +++++++++++++++++------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index dd24639ebcf..318867a5d38 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -503,6 +503,8 @@ class User(Base, Dictifiable, RepresentById): stored_workflows = relationship('StoredWorkflow', back_populates='user', primaryjoin=(lambda: User.id == StoredWorkflow.user_id)) # type: ignore + preferences: association_proxy # defined at the end of this module + # attributes that will be accessed and returned when calling to_dict( view='collection' ) dict_collection_visible_keys = ['id', 'email', 'username', 'deleted', 'active', 'last_password_change'] # attributes that will be accessed and returned when calling to_dict( view='element' ) @@ -2495,10 +2497,13 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, RepresentByI update_time = column_property( select(func.max(HistoryAudit.update_time)).where(HistoryAudit.history_id == id).scalar_subquery(), ) - # `users_shared_with_count` and `average_rating` added at the bottom of this module - # (cannot be added before HistoryUserShareAssociation and HistoryRatingAssociation are defined) - users_shared_with_count: column_property - average_rating: column_property + users_shared_with_count: column_property # defined at the end of this module + average_rating: column_property # defined at the end of this module + + # Set up proxy so that + # History.users_shared_with + # returns a list of users that history is shared with. + users_shared_with_dot_users = association_proxy('users_shared_with', 'user') dict_collection_visible_keys = ['id', 'name', 'published', 'deleted'] dict_element_visible_keys = ['id', 'name', 'genome_build', 'deleted', 'purged', 'update_time', @@ -6251,6 +6256,11 @@ class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): average_rating: column_property + # Set up proxy so that + # StoredWorkflow.users_shared_with + # returns a list of users that workflow is shared with. + users_shared_with_dot_users = association_proxy('users_shared_with', 'user') + dict_collection_visible_keys = ['id', 'name', 'create_time', 'update_time', 'published', 'deleted', 'hidden'] dict_element_visible_keys = ['id', 'name', 'create_time', 'update_time', 'published', 'deleted', 'hidden'] @@ -8182,9 +8192,13 @@ class Page(Base, Dictifiable, RepresentById): users_shared_with = relationship( 'PageUserShareAssociation', back_populates='page') - # `average_rating` added at the bottom of this module - # (cannot be added before PageRatingAssociation is defined) - average_rating: column_property + + average_rating: column_property # defined at the end of this module + + # Set up proxy so that + # Page.users_shared_with + # returns a list of users that page is shared with. + users_shared_with_dot_users = association_proxy('users_shared_with', 'user') dict_element_visible_keys = ['id', 'title', 'latest_revision_id', 'slug', 'published', 'importable', 'deleted', 'username'] @@ -8292,9 +8306,13 @@ class Visualization(Base, RepresentById): ratings = relationship('VisualizationRatingAssociation', order_by='VisualizationRatingAssociation.id', back_populates="visualization") - # `average_rating` added at the bottom of this module - # (cannot be added before VisualizationRatingAssociation is defined) - average_rating: column_property + + average_rating: column_property # defined at the end of this module + + # Set up proxy so that + # Visualization.users_shared_with + # returns a list of users that visualization is shared with. + users_shared_with_dot_users = association_proxy('users_shared_with', 'user') def __init__(self, id=None, user=None, type=None, title=None, dbkey=None, slug=None, latest_revision=None): self.id = id @@ -9315,26 +9333,6 @@ WorkflowInvocationStep.subworkflow_invocation_id = column_property( )).scalar_subquery(), ) -# Set up proxy so that -# History.users_shared_with -# returns a list of users that history is shared with. -History.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore - -# Set up proxy so that -# Page.users_shared_with -# returns a list of users that page is shared with. -Page.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore - -# Set up proxy so that -# StoredWorkflow.users_shared_with -# returns a list of users that workflow is shared with. -StoredWorkflow.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore - # Set up proxy so that this syntax is possible: # .preferences[pref_name] = pref_value -User.preferences = association_proxy('_preferences', 'value', creator=UserPreference) # type: ignore - -# Set up proxy so that -# Visualization.users_shared_with -# returns a list of users that visualization is shared with. -Visualization.users_shared_with_dot_users = association_proxy('users_shared_with', 'user') # type: ignore +User.preferences = association_proxy('_preferences', 'value', creator=UserPreference) From 84b4c27c918bb3ec3b1e797ffcb3ffd4a8738cae Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 12:59:32 -0400 Subject: [PATCH 197/225] Cleanup: breakup very long lines in class attr declarations --- lib/galaxy/model/__init__.py | 175 ++++++++++++++++++++++------------- 1 file changed, 113 insertions(+), 62 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 318867a5d38..a9d31f27241 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -932,7 +932,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): state_history = relationship('JobStateHistory', back_populates='job') text_metrics = relationship('JobMetricText', back_populates='job') numeric_metrics = relationship('JobMetricNumeric', back_populates='job') - job = relationship('GenomeIndexToolData', back_populates='job') # TODO review attr naming (the functionality IS correct) + job = relationship('GenomeIndexToolData', back_populates='job') interactivetool_entry_points = relationship('InteractiveToolEntryPoint', back_populates='job', uselist=True) implicit_collection_jobs_association = relationship('ImplicitCollectionJobsJobAssociation', @@ -1803,7 +1803,8 @@ class JobToOutputDatasetAssociation(Base, RepresentById): job_id = Column(Integer, ForeignKey('job.id'), index=True) dataset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) name = Column(String(255)) - dataset = relationship('HistoryDatasetAssociation', lazy=False, back_populates='creating_job_associations') + dataset = relationship('HistoryDatasetAssociation', + lazy=False, back_populates='creating_job_associations') job = relationship('Job', back_populates='output_datasets') def __init__(self, name, dataset): @@ -1816,7 +1817,8 @@ class JobToInputDatasetCollectionAssociation(Base, RepresentById): id = Column(Integer, primary_key=True) job_id = Column(Integer, ForeignKey('job.id'), index=True) - dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + dataset_collection_id = Column(Integer, + ForeignKey('history_dataset_collection_association.id'), index=True) dataset_version = Column(Integer) name = Column(String(255)) dataset_collection = relationship('HistoryDatasetCollectionAssociation', lazy=False) @@ -1832,7 +1834,8 @@ class JobToInputDatasetCollectionElementAssociation(Base, RepresentById): id = Column(Integer, primary_key=True) job_id = Column(Integer, ForeignKey('job.id'), index=True) - dataset_collection_element_id = Column(Integer, ForeignKey('dataset_collection_element.id'), index=True) + dataset_collection_element_id = Column(Integer, + ForeignKey('dataset_collection_element.id'), index=True) name = Column(Unicode(255)) dataset_collection_element = relationship('DatasetCollectionElement', lazy=False) job = relationship('Job', back_populates='input_dataset_collection_elements') @@ -1849,7 +1852,8 @@ class JobToOutputDatasetCollectionAssociation(Base, RepresentById): id = Column(Integer, primary_key=True) job_id = Column(Integer, ForeignKey('job.id'), index=True) - dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + dataset_collection_id = Column(Integer, + ForeignKey('history_dataset_collection_association.id'), index=True) name = Column(Unicode(255)) dataset_collection_instance = relationship( 'HistoryDatasetCollectionAssociation', @@ -1934,14 +1938,16 @@ class ImplicitlyCreatedDatasetCollectionInput(Base, RepresentById): __tablename__ = 'implicitly_created_dataset_collection_inputs' id = Column(Integer, primary_key=True) - dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) - input_dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + dataset_collection_id = Column(Integer, + ForeignKey('history_dataset_collection_association.id'), index=True) + input_dataset_collection_id = Column(Integer, + ForeignKey('history_dataset_collection_association.id'), index=True) name = Column(Unicode(255)) input_dataset_collection = relationship('HistoryDatasetCollectionAssociation', - primaryjoin=( - lambda: HistoryDatasetCollectionAssociation.id == ImplicitlyCreatedDatasetCollectionInput.input_dataset_collection_id # type: ignore - )) + primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id # type: ignore + == ImplicitlyCreatedDatasetCollectionInput.input_dataset_collection_id) # type: ignore + ) dataset_collection = relationship('HistoryDatasetCollectionAssociation', primaryjoin=(lambda: HistoryDatasetCollectionAssociation.id # type: ignore == ImplicitlyCreatedDatasetCollectionInput.dataset_collection_id), # type: ignore @@ -1957,7 +1963,8 @@ class ImplicitCollectionJobs(Base, RepresentById): id = Column(Integer, primary_key=True) populated_state = Column(TrimmedString(64), default='new', nullable=False) - jobs = relationship('ImplicitCollectionJobsJobAssociation', back_populates='implicit_collection_jobs') + jobs = relationship('ImplicitCollectionJobsJobAssociation', + back_populates='implicit_collection_jobs') history_dataset_collection_associations = relationship('HistoryDatasetCollectionAssociation', back_populates='implicit_collection_jobs') workflow_invocation_step = relationship('WorkflowInvocationStep', @@ -2047,8 +2054,10 @@ class JobExternalOutputMetadata(Base, RepresentById): id = Column(Integer, primary_key=True) job_id = Column(Integer, ForeignKey('job.id'), index=True) - history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True, nullable=True) - library_dataset_dataset_association_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) + history_dataset_association_id = Column(Integer, + ForeignKey('history_dataset_association.id'), index=True, nullable=True) + library_dataset_dataset_association_id = Column(Integer, + ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) is_valid = Column(Boolean, default=True) filename_in = Column(String(255)) filename_out = Column(String(255)) @@ -2308,7 +2317,6 @@ class DeferredJob(Base, RepresentById): state = Column(String(64), index=True) plugin = Column(String(128), index=True) params = Column(MutableJSONType) - # TODO review attr naming: DeferredJob.deferred_job (that points to a GenomeIndexToolData object) is confusing. deferred_job = relationship('GenomeIndexToolData', back_populates='deferred') class states(str, Enum): @@ -2454,7 +2462,8 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, RepresentByI order_by=lambda: desc(JobExportHistoryArchive.id)) # type: ignore active_datasets = relationship('HistoryDatasetAssociation', primaryjoin=( - lambda: and_(HistoryDatasetAssociation.history_id == History.id, not_(HistoryDatasetAssociation.deleted)) # type: ignore + lambda: and_(HistoryDatasetAssociation.history_id # type: ignore + == History.id, not_(HistoryDatasetAssociation.deleted)) # type: ignore ), order_by=lambda: asc(HistoryDatasetAssociation.hid), # type: ignore viewonly=True) @@ -2475,8 +2484,10 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, RepresentByI viewonly=True) visible_dataset_collections = relationship('HistoryDatasetCollectionAssociation', primaryjoin=( - lambda: and_(HistoryDatasetCollectionAssociation.history_id == History.id, # type: ignore - not_(HistoryDatasetCollectionAssociation.deleted), HistoryDatasetCollectionAssociation.visible) # type: ignore + lambda: and_( + HistoryDatasetCollectionAssociation.history_id == History.id, # type: ignore + not_(HistoryDatasetCollectionAssociation.deleted), # type: ignore + HistoryDatasetCollectionAssociation.visible) # type: ignore ), order_by=lambda: asc(HistoryDatasetCollectionAssociation.hid), # type: ignore viewonly=True) @@ -3199,7 +3210,8 @@ class LibraryDatasetDatasetAssociationPermissions(Base, RepresentById): library_dataset_dataset_association_id = Column( Integer, ForeignKey('library_dataset_dataset_association.id'), nullable=True, index=True) role_id = Column(Integer, ForeignKey('role.id'), index=True) - library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', back_populates='actions') + library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', + back_populates='actions') role = relationship('Role', back_populates='library_dataset_dataset_actions') def __init__(self, action, library_item, role): @@ -4457,7 +4469,8 @@ class HistoryDatasetAssociationHistory(Base, RepresentById): __tablename__ = 'history_dataset_association_history' id = Column(Integer, primary_key=True) - history_dataset_association_id = Column(Integer, ForeignKey("history_dataset_association.id"), index=True) + history_dataset_association_id = Column(Integer, + ForeignKey("history_dataset_association.id"), index=True) update_time = Column(DateTime, default=now) version = Column(Integer) name = Column(TrimmedString(255)) @@ -4492,7 +4505,8 @@ class HistoryDatasetAssociationDisplayAtAuthorization(Base, RepresentById): id = Column(Integer, primary_key=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, index=True, default=now, onupdate=now) - history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + history_dataset_association_id = Column(Integer, + ForeignKey('history_dataset_association.id'), index=True) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) site = Column(TrimmedString(255)) history_dataset_association = relationship('HistoryDatasetAssociation') @@ -4508,8 +4522,10 @@ class HistoryDatasetAssociationSubset(Base, RepresentById): __tablename__ = 'history_dataset_association_subset' id = Column(Integer, primary_key=True) - history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) - history_dataset_association_subset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + history_dataset_association_id = Column(Integer, + ForeignKey('history_dataset_association.id'), index=True) + history_dataset_association_subset_id = Column(Integer, + ForeignKey('history_dataset_association.id'), index=True) location = Column(Unicode(255), index=True) hda = relationship('HistoryDatasetAssociation', @@ -4539,7 +4555,7 @@ class Library(Base, Dictifiable, HasName, RepresentById): synopsis = Column(TEXT) root_folder = relationship('LibraryFolder', back_populates='library_root') actions = relationship('LibraryPermissions', back_populates='library') - info_association = relationship('LibraryInfoAssociation', back_populates='library') # TODO: should this be plural? + info_association = relationship('LibraryInfoAssociation', back_populates='library') permitted_actions = get_permitted_actions(filter='LIBRARY') dict_collection_visible_keys = ['id', 'name'] @@ -4736,7 +4752,8 @@ class LibraryDataset(Base, RepresentById): id = Column(Integer, primary_key=True) # current version of dataset, if null, there is not a current version selected library_dataset_dataset_association_id = Column(Integer, - ForeignKey('library_dataset_dataset_association.id', use_alter=True, name='library_dataset_dataset_association_id_fk'), + ForeignKey('library_dataset_dataset_association.id', + use_alter=True, name='library_dataset_dataset_association_id_fk'), nullable=True, index=True) folder_id = Column(Integer, ForeignKey('library_folder.id'), index=True) # not currently being used, but for possible future use @@ -5154,7 +5171,8 @@ class ImplicitlyConvertedDatasetAssociation(Base, RepresentById): create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) hda_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True, nullable=True) - ldda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) + ldda_id = Column(Integer, + ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) hda_parent_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) ldda_parent_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True) deleted = Column(Boolean, index=True, default=False) @@ -5942,11 +5960,15 @@ class DatasetCollectionElement(Base, Dictifiable, RepresentById): id = Column(Integer, primary_key=True) # Parent collection id describing what collection this element belongs to. - dataset_collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True, nullable=False) + dataset_collection_id = Column(Integer, + ForeignKey('dataset_collection.id'), index=True, nullable=False) # Child defined by this association - HDA, LDDA, or another dataset association... - hda_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True, nullable=True) - ldda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) - child_collection_id = Column(Integer, ForeignKey('dataset_collection.id'), index=True, nullable=True) + hda_id = Column(Integer, + ForeignKey('history_dataset_association.id'), index=True, nullable=True) + ldda_id = Column(Integer, + ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) + child_collection_id = Column(Integer, + ForeignKey('dataset_collection.id'), index=True, nullable=True) # Element index and identifier to define this parent-child relationship. element_index = Column(Integer) element_identifier = Column(Unicode(255)) @@ -6215,7 +6237,8 @@ class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): update_time = Column(DateTime, default=now, onupdate=now, index=True) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=False) latest_workflow_id = Column(Integer, - ForeignKey('workflow.id', use_alter=True, name='stored_workflow_latest_workflow_id_fk'), index=True) + ForeignKey('workflow.id', use_alter=True, name='stored_workflow_latest_workflow_id_fk'), + index=True) name = Column(TEXT) deleted = Column(Boolean, default=False) hidden = Column(Boolean, default=False) @@ -6252,7 +6275,8 @@ class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): ratings = relationship('StoredWorkflowRatingAssociation', order_by='StoredWorkflowRatingAssociation.id', back_populates="stored_workflow") - users_shared_with = relationship('StoredWorkflowUserShareAssociation', back_populates='stored_workflow') + users_shared_with = relationship('StoredWorkflowUserShareAssociation', + back_populates='stored_workflow') average_rating: column_property @@ -6914,7 +6938,8 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy back_populates='workflow_invocation') output_datasets = relationship('WorkflowInvocationOutputDatasetAssociation', back_populates='workflow_invocation') - parent_workflow_invocation_association = relationship('WorkflowInvocationToSubworkflowInvocationAssociation', + parent_workflow_invocation_association = relationship( + 'WorkflowInvocationToSubworkflowInvocationAssociation', primaryjoin=(lambda: WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id # type: ignore == WorkflowInvocation.id), # type: ignore @@ -7277,13 +7302,16 @@ class WorkflowInvocationToSubworkflowInvocationAssociation(Base, Dictifiable, Re __tablename__ = 'workflow_invocation_to_subworkflow_invocation_association' id = Column(Integer, primary_key=True) - workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id', name='fk_wfi_swi_wfi'), index=True) - subworkflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id', name='fk_wfi_swi_swi'), index=True) + workflow_invocation_id = Column(Integer, + ForeignKey('workflow_invocation.id', name='fk_wfi_swi_wfi'), index=True) + subworkflow_invocation_id = Column(Integer, + ForeignKey('workflow_invocation.id', name='fk_wfi_swi_swi'), index=True) workflow_step_id = Column(Integer, ForeignKey('workflow_step.id', name='fk_wfi_swi_ws')) subworkflow_invocation = relationship('WorkflowInvocation', primaryjoin=(lambda: - WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id == WorkflowInvocation.id), # type: ignore + WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id # type: ignore + == WorkflowInvocation.id), # type: ignore back_populates='parent_workflow_invocation_association', uselist=False, ) @@ -7317,7 +7345,8 @@ class WorkflowInvocationStep(Base, Dictifiable, RepresentById): workflow_step = relationship('WorkflowStep') job = relationship('Job', back_populates='workflow_invocation_step', uselist=False) - implicit_collection_jobs = relationship('ImplicitCollectionJobs', back_populates='workflow_invocation_step', uselist=False) + implicit_collection_jobs = relationship('ImplicitCollectionJobs', + back_populates='workflow_invocation_step', uselist=False) output_dataset_collections = relationship( 'WorkflowInvocationStepOutputDatasetCollectionAssociation', back_populates='workflow_invocation_step') @@ -7327,7 +7356,8 @@ class WorkflowInvocationStep(Base, Dictifiable, RepresentById): output_value = relationship('WorkflowInvocationOutputValue', foreign_keys='[WorkflowInvocationStep.workflow_invocation_id, WorkflowInvocationStep.workflow_step_id]', primaryjoin=(lambda: and_( - WorkflowInvocationStep.workflow_invocation_id == WorkflowInvocationOutputValue.workflow_invocation_id, # type: ignore + WorkflowInvocationStep.workflow_invocation_id # type: ignore + == WorkflowInvocationOutputValue.workflow_invocation_id, # type: ignore WorkflowInvocationStep.workflow_step_id == WorkflowInvocationOutputValue.workflow_step_id, # type: ignore )), back_populates='workflow_invocation_step', @@ -7492,10 +7522,12 @@ class WorkflowRequestToInputDatasetCollectionAssociation(Base, Dictifiable, Repr name = Column(String(255)) workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id'), index=True) workflow_step_id = Column(Integer, ForeignKey('workflow_step.id')) - dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id'), index=True) + dataset_collection_id = Column(Integer, + ForeignKey('history_dataset_collection_association.id'), index=True) workflow_step = relationship('WorkflowStep') dataset_collection = relationship('HistoryDatasetCollectionAssociation') - workflow_invocation = relationship('WorkflowInvocation', back_populates='input_dataset_collections') + workflow_invocation = relationship('WorkflowInvocation', + back_populates='input_dataset_collections') history_content_type = "dataset_collection" dict_collection_visible_keys = ['id', 'workflow_invocation_id', 'workflow_step_id', 'dataset_collection_id', 'name'] @@ -7541,12 +7573,17 @@ class WorkflowInvocationOutputDatasetCollectionAssociation(Base, Dictifiable, Re __tablename__ = 'workflow_invocation_output_dataset_collection_association' id = Column(Integer, primary_key=True) - workflow_invocation_id = Column(Integer, ForeignKey('workflow_invocation.id', name='fk_wiodca_wii'), index=True) - workflow_step_id = Column(Integer, ForeignKey('workflow_step.id', name='fk_wiodca_wsi'), index=True) - dataset_collection_id = Column(Integer, ForeignKey('history_dataset_collection_association.id', name='fk_wiodca_dci'), index=True) - workflow_output_id = Column(Integer, ForeignKey('workflow_output.id', name='fk_wiodca_woi'), index=True) + workflow_invocation_id = Column(Integer, + ForeignKey('workflow_invocation.id', name='fk_wiodca_wii'), index=True) + workflow_step_id = Column(Integer, + ForeignKey('workflow_step.id', name='fk_wiodca_wsi'), index=True) + dataset_collection_id = Column(Integer, + ForeignKey('history_dataset_collection_association.id', name='fk_wiodca_dci'), index=True) + workflow_output_id = Column(Integer, + ForeignKey('workflow_output.id', name='fk_wiodca_woi'), index=True) - workflow_invocation = relationship('WorkflowInvocation', back_populates='output_dataset_collections') + workflow_invocation = relationship('WorkflowInvocation', + back_populates='output_dataset_collections') workflow_step = relationship('WorkflowStep') dataset_collection = relationship('HistoryDatasetCollectionAssociation') workflow_output = relationship('WorkflowOutput') @@ -7588,10 +7625,12 @@ class WorkflowInvocationStepOutputDatasetAssociation(Base, Dictifiable, Represen __tablename__ = 'workflow_invocation_step_output_dataset_association' id = Column(Integer, primary_key=True) - workflow_invocation_step_id = Column(Integer, ForeignKey('workflow_invocation_step.id'), index=True) + workflow_invocation_step_id = Column(Integer, + ForeignKey('workflow_invocation_step.id'), index=True) dataset_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) output_name = Column(String(255), nullable=True) - workflow_invocation_step = relationship('WorkflowInvocationStep', back_populates='output_datasets') + workflow_invocation_step = relationship('WorkflowInvocationStep', + back_populates='output_datasets') dataset = relationship('HistoryDatasetAssociation') dict_collection_visible_keys = ['id', 'workflow_invocation_step_id', 'dataset_id', 'output_name'] @@ -7607,10 +7646,12 @@ class WorkflowInvocationStepOutputDatasetCollectionAssociation(Base, Dictifiable workflow_step_id = Column( Integer, ForeignKey('workflow_step.id', name='fk_wisodca_wsi'), index=True) dataset_collection_id = Column( - Integer, ForeignKey('history_dataset_collection_association.id', name='fk_wisodca_dci'), index=True) + Integer, + ForeignKey('history_dataset_collection_association.id', name='fk_wisodca_dci'), index=True) output_name = Column(String(255), nullable=True) - workflow_invocation_step = relationship('WorkflowInvocationStep', back_populates='output_dataset_collections') + workflow_invocation_step = relationship('WorkflowInvocationStep', + back_populates='output_dataset_collections') dataset_collection = relationship('HistoryDatasetCollectionAssociation') dict_collection_visible_keys = ['id', 'workflow_invocation_step_id', 'dataset_collection_id', 'output_name'] @@ -7621,8 +7662,10 @@ class MetadataFile(Base, StorableObject, RepresentById): id = Column(Integer, primary_key=True) name = Column(TEXT) - hda_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True, nullable=True) - lda_id = Column(Integer, ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) + hda_id = Column(Integer, + ForeignKey('history_dataset_association.id'), index=True, nullable=True) + lda_id = Column(Integer, + ForeignKey('library_dataset_dataset_association.id'), index=True, nullable=True) create_time = Column(DateTime, default=now) update_time = Column(DateTime, index=True, default=now, onupdate=now) object_store_id = Column(TrimmedString(255), index=True) @@ -7689,8 +7732,8 @@ class FormDefinition(Base, Dictifiable, RepresentById): update_time = Column(DateTime, default=now, onupdate=now) name = Column(TrimmedString(255), nullable=False) desc = Column(TEXT) - form_definition_current_id = Column( - Integer, ForeignKey('form_definition_current.id', use_alter=True), index=True, nullable=False) + form_definition_current_id = Column(Integer, + ForeignKey('form_definition_current.id', use_alter=True), index=True, nullable=False) fields = Column(MutableJSONType) type = Column(TrimmedString(255), index=True) layout = Column(MutableJSONType) @@ -8159,8 +8202,8 @@ class Page(Base, Dictifiable, RepresentById): create_time = Column(DateTime, default=now) update_time = Column(DateTime, default=now, onupdate=now) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True, nullable=False) - latest_revision_id = Column( - Integer, ForeignKey('page_revision.id', use_alter=True, name='page_latest_revision_id_fk'), index=True) + latest_revision_id = Column(Integer, + ForeignKey('page_revision.id', use_alter=True, name='page_latest_revision_id_fk'), index=True) title = Column(TEXT) deleted = Column(Boolean, index=True, default=False) importable = Column(Boolean, index=True, default=False) @@ -8507,7 +8550,8 @@ class HistoryDatasetAssociationTagAssociation(Base, ItemTagAssociation, Represen __tablename__ = 'history_dataset_association_tag_association' id = Column(Integer, primary_key=True) - history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + history_dataset_association_id = Column(Integer, + ForeignKey('history_dataset_association.id'), index=True) tag_id = Column(Integer, ForeignKey('tag.id'), index=True) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) user_tname = Column(TrimmedString(255), index=True) @@ -8663,7 +8707,8 @@ class HistoryDatasetAssociationAnnotationAssociation(Base, RepresentById): ) id = Column(Integer, primary_key=True) - history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + history_dataset_association_id = Column(Integer, + ForeignKey('history_dataset_association.id'), index=True) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) annotation = Column(TEXT) hda = relationship('HistoryDatasetAssociation', back_populates='annotations') @@ -8784,7 +8829,8 @@ class HistoryDatasetAssociationRatingAssociation(ItemRatingAssociation, Represen __tablename__ = 'history_dataset_association_rating_association' id = Column(Integer, primary_key=True) - history_dataset_association_id = Column(Integer, ForeignKey('history_dataset_association.id'), index=True) + history_dataset_association_id = Column(Integer, + ForeignKey('history_dataset_association.id'), index=True) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) rating = Column(Integer, index=True) history_dataset_association = relationship('HistoryDatasetAssociation', back_populates='ratings') @@ -9297,27 +9343,32 @@ Job.any_output_dataset_deleted = column_property( ) History.average_rating = column_property( - select(func.avg(HistoryRatingAssociation.rating)).where(HistoryRatingAssociation.history_id == History.id).scalar_subquery(), + select(func.avg(HistoryRatingAssociation.rating)).where( + HistoryRatingAssociation.history_id == History.id).scalar_subquery(), deferred=True ) History.users_shared_with_count = column_property( - select(func.count(HistoryUserShareAssociation.id)).where(History.id == HistoryUserShareAssociation.history_id).scalar_subquery(), + select(func.count(HistoryUserShareAssociation.id)).where( + History.id == HistoryUserShareAssociation.history_id).scalar_subquery(), deferred=True ) Page.average_rating = column_property( - select(func.avg(PageRatingAssociation.rating)).where(PageRatingAssociation.page_id == Page.id).scalar_subquery(), + select(func.avg(PageRatingAssociation.rating)).where( + PageRatingAssociation.page_id == Page.id).scalar_subquery(), deferred=True ) StoredWorkflow.average_rating = column_property( - select(func.avg(StoredWorkflowRatingAssociation.rating)).where(StoredWorkflowRatingAssociation.stored_workflow_id == StoredWorkflow.id).scalar_subquery(), + select(func.avg(StoredWorkflowRatingAssociation.rating)).where( + StoredWorkflowRatingAssociation.stored_workflow_id == StoredWorkflow.id).scalar_subquery(), deferred=True ) Visualization.average_rating = column_property( - select(func.avg(VisualizationRatingAssociation.rating)).where(VisualizationRatingAssociation.visualization_id == Visualization.id).scalar_subquery(), + select(func.avg(VisualizationRatingAssociation.rating)).where( + VisualizationRatingAssociation.visualization_id == Visualization.id).scalar_subquery(), deferred=True ) From 49a7e3465109b41eb3f405287a07c44f04f21c6b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 13:05:33 -0400 Subject: [PATCH 198/225] Cleanup: all primaryjoin values should be lambdas --- lib/galaxy/model/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index a9d31f27241..a581741c1f0 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -4639,7 +4639,7 @@ class LibraryFolder(Base, Dictifiable, HasName, RepresentById): genome_build = Column(TrimmedString(40)) folders = relationship('LibraryFolder', - primaryjoin=('LibraryFolder.id == LibraryFolder.parent_id'), + primaryjoin=(lambda: LibraryFolder.id == LibraryFolder.parent_id), # type: ignore order_by=asc(name), back_populates='parent') parent = relationship('LibraryFolder', back_populates='folders', remote_side=[id]) @@ -4656,7 +4656,7 @@ class LibraryFolder(Base, Dictifiable, HasName, RepresentById): viewonly=True) datasets = relationship('LibraryDataset', - primaryjoin=('LibraryDataset.folder_id == LibraryFolder.id'), + primaryjoin=(lambda: LibraryDataset.folder_id == LibraryFolder.id), # type: ignore # order_by=asc(text('LibraryDataset._name')), # TODO: uncomment after mapping LibraryDataset lazy=True, viewonly=True) @@ -7740,7 +7740,7 @@ class FormDefinition(Base, Dictifiable, RepresentById): form_definition_current = relationship( 'FormDefinitionCurrent', back_populates='forms', - primaryjoin=('FormDefinitionCurrent.id == FormDefinition.form_definition_current_id')) + primaryjoin=(lambda: FormDefinitionCurrent.id == FormDefinition.form_definition_current_id)) # type: ignore # The following form_builder classes are supported by the FormDefinition class. supported_field_types = [AddressField, CheckboxField, PasswordField, SelectField, TextArea, TextField, WorkflowField, WorkflowMappingField, HistoryField] @@ -7798,11 +7798,11 @@ class FormDefinitionCurrent(Base, RepresentById): 'FormDefinition', back_populates='form_definition_current', cascade='all, delete-orphan', - primaryjoin=('FormDefinitionCurrent.id == FormDefinition.form_definition_current_id')) + primaryjoin=(lambda: FormDefinitionCurrent.id == FormDefinition.form_definition_current_id)) # type: ignore latest_form = relationship( 'FormDefinition', post_update=True, - primaryjoin=('FormDefinitionCurrent.latest_form_id == model.FormDefinition.id')) + primaryjoin=(lambda: FormDefinitionCurrent.latest_form_id == FormDefinition.id)) # type: ignore def __init__(self, form_definition=None): self.latest_form = form_definition @@ -7818,7 +7818,7 @@ class FormValues(Base, RepresentById): content = Column(MutableJSONType) form_definition = relationship( 'FormDefinition', - primaryjoin=('FormValues.form_definition_id == FormDefinition.id')) + primaryjoin=(lambda: FormValues.form_definition_id == FormDefinition.id)) # type: ignore def __init__(self, form_def=None, content=None): self.form_definition = form_def @@ -8213,12 +8213,12 @@ class Page(Base, Dictifiable, RepresentById): revisions = relationship( 'PageRevision', cascade="all, delete-orphan", - primaryjoin=('Page.id == PageRevision.page_id'), + primaryjoin=(lambda: Page.id == PageRevision.page_id), # type: ignore back_populates='page') latest_revision = relationship( 'PageRevision', post_update=True, - primaryjoin=('Page.latest_revision_id == PageRevision.id'), + primaryjoin=(lambda: Page.latest_revision_id == PageRevision.id), # type: ignore lazy=False) tags = relationship( 'PageTagAssociation', @@ -8279,9 +8279,8 @@ class PageRevision(Base, Dictifiable, RepresentById): title = Column(TEXT) content = Column(TEXT) content_format = Column(TrimmedString(32)) - page = relationship( - 'Page', - primaryjoin=('Page.id == PageRevision.page_id')) + page = relationship('Page', + primaryjoin=(lambda: Page.id == PageRevision.page_id)) # type: ignore DEFAULT_CONTENT_FORMAT = 'html' dict_element_visible_keys = ['id', 'page_id', 'title', 'content', 'content_format'] From 5a675796cc0b97ef6068a405e382b5406ba15611 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 13:15:41 -0400 Subject: [PATCH 199/225] Cleanup: all order_by values should be lambdas --- lib/galaxy/model/__init__.py | 50 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index a581741c1f0..821dd003b1f 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -458,21 +458,21 @@ class User(Base, Dictifiable, RepresentById): addresses = relationship('UserAddress', back_populates='user', - order_by='desc(UserAddress.update_time)') + order_by=lambda: desc(UserAddress.update_time)) # type: ignore cloudauthz = relationship('CloudAuthz', back_populates='user') custos_auth = relationship('CustosAuthnzToken', back_populates='user') default_permissions = relationship('DefaultUserPermissions', back_populates='user') groups = relationship('UserGroupAssociation', back_populates='user') histories = relationship('History', backref='user', - order_by='desc(History.update_time)') + order_by=lambda: desc(History.update_time)) # type: ignore active_histories = relationship('History', primaryjoin=(lambda: (History.user_id == User.id) & (not_(History.deleted))), # type: ignore viewonly=True, - order_by='desc(History.update_time)') + order_by=lambda: desc(History.update_time)) # type: ignore galaxy_sessions = relationship('GalaxySession', backref='user', - order_by='desc(GalaxySession.update_time)') + order_by=lambda: desc(GalaxySession.update_time)) # type: ignore pages_shared_by_others = relationship('PageUserShareAssociation', back_populates='user') quotas = relationship('UserQuotaAssociation', back_populates='user') social_auth = relationship('UserAuthnzToken', back_populates='user') @@ -493,7 +493,7 @@ class User(Base, Dictifiable, RepresentById): # Add type hint (will this work w/SA?) api_keys: 'List[APIKeys]' = relationship('APIKeys', back_populates='user', - order_by='desc(APIKeys.create_time)') + order_by=lambda: desc(APIKeys.create_time)) # type: ignore pages = relationship('Page', back_populates='user') reset_tokens = relationship('PasswordResetToken', back_populates='user') histories_shared_by_others = relationship('HistoryUserShareAssociation', back_populates='user') @@ -4657,14 +4657,14 @@ class LibraryFolder(Base, Dictifiable, HasName, RepresentById): datasets = relationship('LibraryDataset', primaryjoin=(lambda: LibraryDataset.folder_id == LibraryFolder.id), # type: ignore - # order_by=asc(text('LibraryDataset._name')), # TODO: uncomment after mapping LibraryDataset + order_by=(lambda: asc(LibraryDataset._name)), # type: ignore lazy=True, viewonly=True) active_datasets = relationship('LibraryDataset', primaryjoin=( 'and_(LibraryDataset.folder_id == LibraryFolder.id, not_(LibraryDataset.deleted))'), - # order_by=asc(text('LibraryDataset._name')), # TODO: uncomment after mapping LibraryDataset + order_by=(lambda: asc(LibraryDataset._name)), # type: ignore lazy=True, viewonly=True) @@ -5243,7 +5243,7 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, RepresentById): elements = relationship('DatasetCollectionElement', primaryjoin=(lambda: DatasetCollection.id == DatasetCollectionElement.dataset_collection_id), # type: ignore back_populates='collection', - order_by='DatasetCollectionElement.element_index') + order_by=lambda: DatasetCollectionElement.element_index) # type: ignore output_dataset_collections = relationship( 'JobToImplicitOutputDatasetCollectionAssociation', back_populates='dataset_collection') @@ -5680,17 +5680,17 @@ class HistoryDatasetCollectionAssociation( ) tags = relationship( 'HistoryDatasetCollectionTagAssociation', - order_by='HistoryDatasetCollectionTagAssociation.id', + order_by=lambda: HistoryDatasetCollectionTagAssociation.id, # type: ignore back_populates='dataset_collection', ) annotations = relationship( 'HistoryDatasetCollectionAssociationAnnotationAssociation', - order_by='HistoryDatasetCollectionAssociationAnnotationAssociation.id', + order_by=lambda: HistoryDatasetCollectionAssociationAnnotationAssociation.id, # type: ignore back_populates='history_dataset_collection', ) ratings = relationship( 'HistoryDatasetCollectionRatingAssociation', - order_by='HistoryDatasetCollectionRatingAssociation.id', + order_by=lambda: HistoryDatasetCollectionRatingAssociation.id, # type: ignore back_populates='dataset_collection', ) output_dataset_collection_instances = relationship( @@ -5919,13 +5919,13 @@ class LibraryDatasetCollectionAssociation(Base, DatasetCollectionInstance, Repre folder = relationship('LibraryFolder', back_populates='dataset_collections') tags = relationship('LibraryDatasetCollectionTagAssociation', - order_by='LibraryDatasetCollectionTagAssociation.id', + order_by=lambda: LibraryDatasetCollectionTagAssociation.id, # type: ignore back_populates='dataset_collection') annotations = relationship('LibraryDatasetCollectionAnnotationAssociation', - order_by='LibraryDatasetCollectionAnnotationAssociation.id', + order_by=lambda: LibraryDatasetCollectionAnnotationAssociation.id, # type: ignore back_populates="dataset_collection") ratings = relationship('LibraryDatasetCollectionRatingAssociation', - order_by='LibraryDatasetCollectionRatingAssociation.id', + order_by=lambda: LibraryDatasetCollectionRatingAssociation.id, # type: ignore back_populates="dataset_collection") editable_keys = ('name', 'deleted') @@ -6254,13 +6254,13 @@ class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): back_populates='stored_workflow', cascade="all, delete-orphan", primaryjoin=(lambda: StoredWorkflow.id == Workflow.stored_workflow_id), # type: ignore - order_by='-Workflow.id') + order_by=lambda: -Workflow.id) # type: ignore latest_workflow = relationship('Workflow', post_update=True, primaryjoin=(lambda: StoredWorkflow.latest_workflow_id == Workflow.id), # type: ignore lazy=False) tags = relationship('StoredWorkflowTagAssociation', - order_by='StoredWorkflowTagAssociation.id', + order_by=lambda: StoredWorkflowTagAssociation.id, # type: ignore back_populates="stored_workflow") owner_tags = relationship('StoredWorkflowTagAssociation', primaryjoin=(lambda: @@ -6268,12 +6268,12 @@ class StoredWorkflow(Base, HasTags, Dictifiable, RepresentById): StoredWorkflow.user_id == StoredWorkflowTagAssociation.user_id) # type: ignore ), viewonly=True, - order_by='StoredWorkflowTagAssociation.id') + order_by=lambda: StoredWorkflowTagAssociation.id) # type: ignore annotations = relationship('StoredWorkflowAnnotationAssociation', - order_by='StoredWorkflowAnnotationAssociation.id', + order_by=lambda: StoredWorkflowAnnotationAssociation.id, # type: ignore back_populates="stored_workflow") ratings = relationship('StoredWorkflowRatingAssociation', - order_by='StoredWorkflowRatingAssociation.id', + order_by=lambda: StoredWorkflowRatingAssociation.id, # type: ignore back_populates="stored_workflow") users_shared_with = relationship('StoredWorkflowUserShareAssociation', back_populates='stored_workflow') @@ -8222,15 +8222,15 @@ class Page(Base, Dictifiable, RepresentById): lazy=False) tags = relationship( 'PageTagAssociation', - order_by='PageTagAssociation.id', + order_by=lambda: PageTagAssociation.id, # type: ignore back_populates='page') annotations = relationship( 'PageAnnotationAssociation', - order_by='PageAnnotationAssociation.id', + order_by=lambda: PageAnnotationAssociation.id, # type: ignore back_populates='page') ratings = relationship( 'PageRatingAssociation', - order_by='PageRatingAssociation.id', + order_by=lambda: PageRatingAssociation.id, # type: ignore back_populates='page') users_shared_with = relationship( 'PageUserShareAssociation', @@ -8340,13 +8340,13 @@ class Visualization(Base, RepresentById): primaryjoin=(lambda: Visualization.latest_revision_id == VisualizationRevision.id), # type: ignore lazy=False) tags = relationship('VisualizationTagAssociation', - order_by='VisualizationTagAssociation.id', + order_by=lambda: VisualizationTagAssociation.id, # type: ignore back_populates="visualization") annotations = relationship('VisualizationAnnotationAssociation', - order_by='VisualizationAnnotationAssociation.id', + order_by=lambda: VisualizationAnnotationAssociation.id, # type: ignore back_populates="visualization") ratings = relationship('VisualizationRatingAssociation', - order_by='VisualizationRatingAssociation.id', + order_by=lambda: VisualizationRatingAssociation.id, # type: ignore back_populates="visualization") average_rating: column_property # defined at the end of this module From 0ebf25e118184d680670d9adaf1cfcb6e0edcef1 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 14:23:27 -0400 Subject: [PATCH 200/225] Cleanup: update docstring in mapping.py --- lib/galaxy/model/mapping.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index ce5ef43327d..d4b32513a99 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1,6 +1,7 @@ """ -Details of how the data model objects are mapped onto the relational database -are encapsulated here. +This module no longer contains the mapping of data model classes to the +relational database. +The module will be revised during migration from SQLAlchemy Migrate to Alembic. """ import logging From ba8844f5c487251d9d4cd7e2e0dba834a483cf3d Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 19:28:20 -0400 Subject: [PATCH 201/225] Replace last backrefs with back_populates --- lib/galaxy/model/__init__.py | 99 +++++++++++++++++++++++++++-------- test/unit/model/test_model.py | 64 +++++++++++++++++++--- 2 files changed, 134 insertions(+), 29 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 821dd003b1f..b26f2cc7b8c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -464,14 +464,14 @@ class User(Base, Dictifiable, RepresentById): default_permissions = relationship('DefaultUserPermissions', back_populates='user') groups = relationship('UserGroupAssociation', back_populates='user') histories = relationship('History', - backref='user', + back_populates='user', order_by=lambda: desc(History.update_time)) # type: ignore active_histories = relationship('History', primaryjoin=(lambda: (History.user_id == User.id) & (not_(History.deleted))), # type: ignore viewonly=True, order_by=lambda: desc(History.update_time)) # type: ignore galaxy_sessions = relationship('GalaxySession', - backref='user', + back_populates='user', order_by=lambda: desc(GalaxySession.update_time)) # type: ignore pages_shared_by_others = relationship('PageUserShareAssociation', back_populates='user') quotas = relationship('UserQuotaAssociation', back_populates='user') @@ -482,11 +482,11 @@ class User(Base, Dictifiable, RepresentById): & (StoredWorkflowMenuEntry.stored_workflow_id == StoredWorkflow.id) # type: ignore & not_(StoredWorkflow.deleted) # type: ignore ), - backref='user', + back_populates='user', cascade='all, delete-orphan', collection_class=ordering_list('order_index')) _preferences = relationship('UserPreference', - backref='user', + back_populates='user', collection_class=attribute_mapped_collection('name')) values = relationship('FormValues', primaryjoin=(lambda: User.form_values_id == FormValues.id)) # type: ignore @@ -502,6 +502,8 @@ class User(Base, Dictifiable, RepresentById): roles = relationship('UserRoleAssociation', back_populates='user') stored_workflows = relationship('StoredWorkflow', back_populates='user', primaryjoin=(lambda: User.id == StoredWorkflow.user_id)) # type: ignore + visualizations_shared_by_others = relationship('VisualizationUserShareAssociation', + back_populates='user') preferences: association_proxy # defined at the end of this module @@ -909,7 +911,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): user = relationship('User') galaxy_session = relationship('GalaxySession') - history = relationship('History', backref='jobs') + history = relationship('History', back_populates='jobs') library_folder = relationship('LibraryFolder', lazy=True) parameters = relationship('JobParameter', lazy=True) input_datasets = relationship('JobToInputDatasetAssociation', back_populates='job') @@ -921,12 +923,13 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): back_populates='job', lazy=True) output_dataset_collections = relationship('JobToImplicitOutputDatasetCollectionAssociation', back_populates='job', lazy=True) - post_job_actions = relationship('PostJobActionAssociation', backref='job', lazy=False) + post_job_actions = relationship('PostJobActionAssociation', back_populates='job', lazy=False) input_library_datasets = relationship('JobToInputLibraryDatasetAssociation', back_populates='job') output_library_datasets = relationship('JobToOutputLibraryDatasetAssociation', back_populates='job', lazy=True) - external_output_metadata = relationship('JobExternalOutputMetadata', lazy=True, backref='job') + external_output_metadata = relationship('JobExternalOutputMetadata', + lazy=True, back_populates='job') tasks = relationship('Task', back_populates='job') output_datasets = relationship('JobToOutputDatasetAssociation', back_populates='job') state_history = relationship('JobStateHistory', back_populates='job') @@ -2038,6 +2041,7 @@ class PostJobActionAssociation(Base, RepresentById): job_id = Column(Integer, ForeignKey('job.id'), index=True, nullable=False) post_job_action_id = Column(Integer, ForeignKey('post_job_action.id'), index=True, nullable=False) post_job_action = relationship('PostJobAction') + job = relationship('Job', back_populates='post_job_actions') def __init__(self, pja, job=None, job_id=None): if job is not None: @@ -2067,6 +2071,7 @@ class JobExternalOutputMetadata(Base, RepresentById): job_runner_external_pid = Column(String(255)) history_dataset_association = relationship('HistoryDatasetAssociation', lazy=False) library_dataset_dataset_association = relationship('LibraryDatasetDatasetAssociation', lazy=False) + job = relationship('Job', back_populates='external_output_metadata') def __init__(self, job=None, dataset=None): self.job = job @@ -2504,6 +2509,8 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, RepresentByI users_shared_with = relationship('HistoryUserShareAssociation', back_populates='history') galaxy_sessions = relationship('GalaxySessionToHistoryAssociation', back_populates='history') workflow_invocations = relationship('WorkflowInvocation', back_populates='history') + user = relationship('User', back_populates='histories') + jobs = relationship('Job', back_populates='history') update_time = column_property( select(func.max(HistoryAudit.update_time)).where(HistoryAudit.history_id == id).scalar_subquery(), @@ -2941,7 +2948,7 @@ class UserRoleAssociation(Base, RepresentById): user = relationship('User', back_populates="roles") role = relationship('Role', back_populates="users") - # TODO: this should be defined on the User model only + # TODO: should be defined on the User model only? non_private_roles = relationship( 'User', backref="non_private_roles", @@ -5144,7 +5151,7 @@ class LibraryDatasetDatasetInfoAssociation(Base, RepresentById): == LibraryDatasetDatasetAssociation.id) & (not_(LibraryDatasetDatasetInfoAssociation.deleted)) ), - backref="info_association") + back_populates="info_association") template = relationship('FormDefinition', primaryjoin=(lambda: @@ -5191,6 +5198,10 @@ class ImplicitlyConvertedDatasetAssociation(Base, RepresentById): primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.hda_id # type: ignore == HistoryDatasetAssociation.id), # type: ignore back_populates='implicitly_converted_parent_datasets') + parent_ldda = relationship('LibraryDatasetDatasetAssociation', + primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_parent_id # type: ignore + == LibraryDatasetDatasetAssociation.table.c.id), # type: ignore + back_populates='implicitly_converted_datasets') def __init__(self, id=None, parent=None, dataset=None, file_type=None, deleted=False, purged=False, metadata_safe=True): self.id = id @@ -5697,6 +5708,10 @@ class HistoryDatasetCollectionAssociation( 'JobToOutputDatasetCollectionAssociation', back_populates='dataset_collection_instance', ) + hidden_dataset_instances = relationship('HistoryDatasetAssociation', + primaryjoin=(lambda: HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id # type: ignore + == HistoryDatasetCollectionAssociation.id), # type: ignore + back_populates='hidden_beneath_collection_instance') editable_keys = ('name', 'deleted', 'visible') @@ -6158,6 +6173,7 @@ class GalaxySession(Base, RepresentById): last_action = Column(DateTime) current_history = relationship('History') histories = relationship('GalaxySessionToHistoryAssociation', back_populates='galaxy_session') + user = relationship('User', back_populates='galaxy_sessions') def __init__(self, id=None, @@ -6352,7 +6368,7 @@ class Workflow(Base, Dictifiable, RepresentById): uuid = Column(UUIDType, nullable=True) steps = relationship('WorkflowStep', - backref='workflow', + back_populates='workflow', primaryjoin=(lambda: Workflow.id == WorkflowStep.workflow_id), # type: ignore order_by=lambda: asc(WorkflowStep.order_index), # type: ignore cascade="all, delete-orphan", @@ -6545,6 +6561,10 @@ class WorkflowStep(Base, RepresentById): output_connections = relationship('WorkflowStepConnection', primaryjoin=(lambda: WorkflowStepConnection.output_step_id == WorkflowStep.id) # type: ignore ) + workflow = relationship('Workflow', + primaryjoin=(lambda: Workflow.id == WorkflowStep.workflow_id), # type: ignore + back_populates='steps' + ) STEP_TYPE_TO_INPUT_TYPE = { "data_input": "dataset", @@ -6895,6 +6915,12 @@ class StoredWorkflowMenuEntry(Base, RepresentById): order_index = Column(Integer) stored_workflow = relationship('StoredWorkflow') + user = relationship('User', back_populates='stored_workflow_menu_entries', + primaryjoin=(lambda: + (StoredWorkflowMenuEntry.user_id == User.id) # type: ignore + & (StoredWorkflowMenuEntry.stored_workflow_id == StoredWorkflow.id) # type: ignore + & not_(StoredWorkflow.deleted)) # type: ignore + ) def __init__(self): self.stored_workflow = None @@ -6944,6 +6970,7 @@ class WorkflowInvocation(Base, UsesCreateAndUpdateTime, Dictifiable, RepresentBy WorkflowInvocationToSubworkflowInvocationAssociation.subworkflow_invocation_id # type: ignore == WorkflowInvocation.id), # type: ignore back_populates='subworkflow_invocation') + output_values = relationship('WorkflowInvocationOutputValue', back_populates='workflow_invocation') dict_collection_visible_keys = ['id', 'update_time', 'create_time', 'workflow_id', 'history_id', 'uuid', 'state'] dict_element_visible_keys = ['id', 'update_time', 'create_time', 'workflow_id', 'history_id', 'uuid', 'state'] @@ -7602,7 +7629,7 @@ class WorkflowInvocationOutputValue(Base, Dictifiable, RepresentById): workflow_output_id = Column(Integer, ForeignKey('workflow_output.id'), index=True) value = Column(MutableJSONType) - workflow_invocation = relationship('WorkflowInvocation', backref="output_values") + workflow_invocation = relationship('WorkflowInvocation', back_populates="output_values") workflow_invocation_step = relationship('WorkflowInvocationStep', foreign_keys='[WorkflowInvocationStep.workflow_invocation_id, WorkflowInvocationStep.workflow_step_id]', @@ -8348,6 +8375,7 @@ class Visualization(Base, RepresentById): ratings = relationship('VisualizationRatingAssociation', order_by=lambda: VisualizationRatingAssociation.id, # type: ignore back_populates="visualization") + users_shared_with = relationship('VisualizationUserShareAssociation', back_populates='visualization') average_rating: column_property # defined at the end of this module @@ -8435,8 +8463,8 @@ class VisualizationUserShareAssociation(Base, UserShareAssociation): id = Column(Integer, primary_key=True) visualization_id = Column(Integer, ForeignKey('visualization.id'), index=True) user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) - user = relationship('User', backref='visualizations_shared_by_others') - visualization = relationship('Visualization', backref='users_shared_with') + user = relationship('User', back_populates='visualizations_shared_by_others') + visualization = relationship('Visualization', back_populates='users_shared_with') def __init__(self): self.visualization = None @@ -8955,6 +8983,7 @@ class UserPreference(Base, RepresentById): user_id = Column(Integer, ForeignKey('galaxy_user.id'), index=True) name = Column(Unicode(255), index=True) value = Column(Text) + user = relationship('User', back_populates='_preferences') def __init__(self, name=None, value=None): self.name = name @@ -9221,6 +9250,10 @@ mapper_registry.map_imperatively( sources=relationship(DatasetSource, back_populates='dataset'), job_export_history_archive=relationship(JobExportHistoryArchive, back_populates='dataset'), genome_index_tool_data=relationship(GenomeIndexToolData, back_populates='dataset'), + history_associations=relationship(HistoryDatasetAssociation, back_populates='dataset'), + library_associations=relationship(LibraryDatasetDatasetAssociation, + primaryjoin=(LibraryDatasetDatasetAssociation.table.c.dataset_id == Dataset.table.c.id), + back_populates='dataset'), ) ) @@ -9231,17 +9264,25 @@ mapper_registry.map_imperatively( dataset=relationship(Dataset, primaryjoin=(Dataset.table.c.id == HistoryDatasetAssociation.table.c.dataset_id), lazy=False, - backref='history_associations'), + back_populates='history_associations'), copied_from_history_dataset_association=relationship(HistoryDatasetAssociation, primaryjoin=(HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id == HistoryDatasetAssociation.table.c.id), remote_side=[HistoryDatasetAssociation.table.c.id], uselist=False, - backref='copied_to_history_dataset_associations'), + back_populates='copied_to_history_dataset_associations'), + copied_from_library_dataset_dataset_association=relationship(LibraryDatasetDatasetAssociation, + primaryjoin=(LibraryDatasetDatasetAssociation.table.c.id + == HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id), + back_populates='copied_to_history_dataset_associations'), + copied_to_history_dataset_associations=relationship(HistoryDatasetAssociation, + primaryjoin=(HistoryDatasetAssociation.table.c.copied_from_history_dataset_association_id + == HistoryDatasetAssociation.table.c.id), + back_populates='copied_from_history_dataset_association'), copied_to_library_dataset_dataset_associations=relationship(LibraryDatasetDatasetAssociation, primaryjoin=(HistoryDatasetAssociation.table.c.id == LibraryDatasetDatasetAssociation.table.c.copied_from_history_dataset_association_id), - backref='copied_from_history_dataset_association'), + back_populates='copied_from_history_dataset_association'), tags=relationship(HistoryDatasetAssociationTagAssociation, order_by=HistoryDatasetAssociationTagAssociation.id, back_populates='history_dataset_association'), @@ -9258,7 +9299,7 @@ mapper_registry.map_imperatively( primaryjoin=(HistoryDatasetAssociation.table.c.hidden_beneath_collection_instance_id == HistoryDatasetCollectionAssociation.id), uselist=False, - backref="hidden_dataset_instances"), + back_populates="hidden_dataset_instances"), _metadata=deferred(HistoryDatasetAssociation.table.c._metadata), dependent_jobs=relationship(JobToInputDatasetAssociation, back_populates='dataset'), creating_job_associations=relationship( @@ -9281,7 +9322,7 @@ mapper_registry.map_imperatively( properties=dict( dataset=relationship(Dataset, primaryjoin=(LibraryDatasetDatasetAssociation.table.c.dataset_id == Dataset.table.c.id), - backref='library_associations'), + back_populates='library_associations'), library_dataset=relationship(LibraryDataset, foreign_keys=LibraryDatasetDatasetAssociation.table.c.library_dataset_id), user=relationship(User), @@ -9291,15 +9332,20 @@ mapper_registry.map_imperatively( == LibraryDatasetDatasetAssociation.table.c.id), remote_side=[LibraryDatasetDatasetAssociation.table.c.id], uselist=False, - backref='copied_to_library_dataset_dataset_associations'), + back_populates='copied_to_library_dataset_dataset_associations'), + copied_to_library_dataset_dataset_associations=relationship(LibraryDatasetDatasetAssociation, + primaryjoin=( + LibraryDatasetDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id + == LibraryDatasetDatasetAssociation.table.c.id), + back_populates='copied_from_library_dataset_dataset_association'), copied_to_history_dataset_associations=relationship(HistoryDatasetAssociation, primaryjoin=(LibraryDatasetDatasetAssociation.table.c.id == HistoryDatasetAssociation.table.c.copied_from_library_dataset_dataset_association_id), - backref='copied_from_library_dataset_dataset_association'), + back_populates='copied_from_library_dataset_dataset_association'), implicitly_converted_datasets=relationship(ImplicitlyConvertedDatasetAssociation, primaryjoin=(ImplicitlyConvertedDatasetAssociation.ldda_parent_id == LibraryDatasetDatasetAssociation.table.c.id), - backref='parent_ldda'), + back_populates='parent_ldda'), tags=relationship(LibraryDatasetDatasetAssociationTagAssociation, order_by=LibraryDatasetDatasetAssociationTagAssociation.id, back_populates='library_dataset_dataset_association'), @@ -9319,6 +9365,17 @@ mapper_registry.map_imperatively( primaryjoin=(lambda: ImplicitlyConvertedDatasetAssociation.ldda_id == LibraryDatasetDatasetAssociation.id), back_populates='dataset_ldda'), + copied_from_history_dataset_association=relationship(HistoryDatasetAssociation, + primaryjoin=(HistoryDatasetAssociation.table.c.id + == LibraryDatasetDatasetAssociation.table.c.copied_from_history_dataset_association_id), + back_populates='copied_to_library_dataset_dataset_associations'), + info_association=relationship(LibraryDatasetDatasetInfoAssociation, + primaryjoin=(lambda: + (LibraryDatasetDatasetInfoAssociation.library_dataset_dataset_association_id + == LibraryDatasetDatasetAssociation.id) + & (not_(LibraryDatasetDatasetInfoAssociation.deleted)) + ), + back_populates='library_dataset_dataset_association'), ) ) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index a47fc4c4ff5..182a394cce9 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -377,8 +377,9 @@ class TestDataManagerJobAssociation(BaseTest): class TestDataset(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'dataset' + def test_table(self, cls_): + # mapped imperatively, so do not test cls_.__tablename__ + assert cls_.table.name == 'dataset' def test_columns(self, session, cls_, job): create_time = datetime.now() @@ -459,6 +460,9 @@ class TestDataset(BaseTest): assert stored_obj.active_library_associations == [library_dataset_dataset_association] assert stored_obj.hashes == [dataset_hash] assert stored_obj.sources == [dataset_source] + assert stored_obj.library_associations == [library_dataset_dataset_association] + assert len(stored_obj.history_associations) == 2 + # TODO: test membership in history_associations by id class TestDatasetCollection(BaseTest): @@ -1289,6 +1293,7 @@ class TestHistory(BaseTest): history_user_share_association, galaxy_session_history_association, workflow_invocation, + job, ): obj = cls_() obj.user = user @@ -1305,6 +1310,7 @@ class TestHistory(BaseTest): obj.users_shared_with.append(history_user_share_association) obj.galaxy_sessions.append(galaxy_session_history_association) obj.workflow_invocations.append(workflow_invocation) + obj.jobs.append(job) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -1325,6 +1331,7 @@ class TestHistory(BaseTest): assert stored_obj.default_permissions == [default_history_permissions] assert stored_obj.galaxy_sessions == [galaxy_session_history_association] assert stored_obj.workflow_invocations == [workflow_invocation] + assert stored_obj.jobs == [job] def test_average_rating(self, session, history, user, history_rating_association_factory): # History has been expunged; to access its deferred properties, @@ -1344,8 +1351,9 @@ class TestHistory(BaseTest): class TestHistoryDatasetAssociation(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'history_dataset_association' + def test_table(self, cls_): + # mapped imperatively, so do not test cls_.__tablename__ + assert cls_.table.name == 'history_dataset_association' def test_columns( self, @@ -1456,9 +1464,9 @@ class TestHistoryDatasetAssociation(BaseTest): library_dataset_dataset_association_factory, ): copied_from_hda = history_dataset_association_factory() + copied_to_hda = history_dataset_association_factory() copied_from_ldda = library_dataset_dataset_association_factory() copied_to_ldda = library_dataset_dataset_association_factory() - icda = implicitly_converted_dataset_association_factory() icpda = implicitly_converted_dataset_association_factory() @@ -1471,6 +1479,7 @@ class TestHistoryDatasetAssociation(BaseTest): obj.validated_state = 'a' obj.hidden_beneath_collection_instance = history_dataset_collection_association obj.copied_to_library_dataset_dataset_associations.append(copied_to_ldda) + obj.copied_to_history_dataset_associations.append(copied_to_hda) obj.tags.append(history_dataset_association_tag_association) obj.annotations.append(history_dataset_association_annotation_association) obj.ratings.append(history_dataset_association_rating_association) @@ -1489,6 +1498,7 @@ class TestHistoryDatasetAssociation(BaseTest): assert stored_obj.extended_metadata.id == extended_metadata.id assert stored_obj.hidden_beneath_collection_instance.id == history_dataset_collection_association.id assert stored_obj.copied_to_library_dataset_dataset_associations == [copied_to_ldda] + assert stored_obj.copied_to_history_dataset_associations == [copied_to_hda] assert stored_obj.tags == [history_dataset_association_tag_association] assert stored_obj.annotations == [history_dataset_association_annotation_association] assert stored_obj.ratings == [history_dataset_association_rating_association] @@ -1840,6 +1850,7 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): history_dataset_collection_rating_association, history_dataset_collection_tag_association, job_to_output_dataset_collection_association, + history_dataset_association, ): copied_to_hdca = history_dataset_collection_association_factory() @@ -1855,6 +1866,7 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): obj.annotations.append(history_dataset_collection_annotation_association) obj.ratings.append(history_dataset_collection_rating_association) obj.output_dataset_collection_instances.append(job_to_output_dataset_collection_association) + obj.hidden_dataset_instances.append(history_dataset_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -1871,6 +1883,7 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): assert stored_obj.ratings == [history_dataset_collection_rating_association] assert stored_obj.output_dataset_collection_instances == [job_to_output_dataset_collection_association] assert stored_obj.job_state_summary # this is a view; TODO: can we test this better? + assert stored_obj.hidden_dataset_instances == [history_dataset_association] class TestHistoryDatasetCollectionAssociationAnnotationAssociation(BaseTest): @@ -3185,8 +3198,9 @@ class TestLibraryDatasetCollectionRatingAssociation(BaseTest): class TestLibraryDatasetDatasetAssociation(BaseTest): - # def test_table(self, cls_): - # assert cls_.__tablename__ == 'library_dataset_dataset_association' + def test_table(self, cls_): + # mapped imperatively, so do not test cls_.__tablename__ + assert cls_.table.name == 'library_dataset_dataset_association' def test_columns( self, @@ -3292,13 +3306,15 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): implicitly_converted_dataset_association_factory, job_to_input_library_dataset_association, job_to_output_library_dataset_association, + library_dataset_dataset_info_association, ): copied_from_ldda = library_dataset_dataset_association_factory() + copied_to_ldda = library_dataset_dataset_association_factory() parent = library_dataset_dataset_association_factory() copied_to_hda = history_dataset_association_factory() icda = implicitly_converted_dataset_association_factory() icpda = implicitly_converted_dataset_association_factory() - persisted = [copied_from_ldda, parent, copied_to_hda, icda, icpda] + persisted = [copied_from_ldda, copied_to_ldda, parent, copied_to_hda, icda, icpda] obj = cls_() obj.library_dataset = library_dataset @@ -3310,10 +3326,12 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): obj.tags.append(library_dataset_dataset_association_tag_association) obj.actions.append(library_dataset_dataset_association_permission) obj.copied_to_history_dataset_associations.append(copied_to_hda) + obj.copied_to_library_dataset_dataset_associations.append(copied_to_ldda) obj.implicitly_converted_datasets.append(icda) obj.implicitly_converted_parent_datasets.append(icpda) obj.dependent_jobs.append(job_to_input_library_dataset_association) obj.creating_job_associations.append(job_to_output_library_dataset_association) + obj.info_association.append(library_dataset_dataset_info_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -3326,10 +3344,12 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): assert stored_obj.tags == [library_dataset_dataset_association_tag_association] assert stored_obj.actions == [library_dataset_dataset_association_permission] assert stored_obj.copied_to_history_dataset_associations == [copied_to_hda] + assert stored_obj.copied_to_library_dataset_dataset_associations == [copied_to_ldda] assert stored_obj.implicitly_converted_datasets == [icda] assert stored_obj.implicitly_converted_parent_datasets == [icpda] assert stored_obj.dependent_jobs == [job_to_input_library_dataset_association] assert stored_obj.creating_job_associations == [job_to_output_library_dataset_association] + assert stored_obj.info_association == [library_dataset_dataset_info_association] delete_from_database(session, persisted) @@ -4807,6 +4827,7 @@ class TestUser(BaseTest): user_role_association, stored_workflow, stored_workflow_menu_entry_factory, + visualization_user_share_association, ): history1 = history_factory(deleted=False) history2 = history_factory(deleted=True) @@ -4844,6 +4865,7 @@ class TestUser(BaseTest): obj.workflows_shared_by_others.append(stored_workflow_user_share_association) obj.roles.append(user_role_association) obj.stored_workflows.append(stored_workflow) + obj.visualizations_shared_by_others.append(visualization_user_share_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -4873,6 +4895,7 @@ class TestUser(BaseTest): assert stored_obj.workflows_shared_by_others == [stored_workflow_user_share_association] assert stored_obj.roles == [user_role_association] assert stored_obj.stored_workflows == [stored_workflow] + assert stored_obj.visualizations_shared_by_others == [visualization_user_share_association] delete_from_database(session, [history1, history2, swme]) @@ -5144,6 +5167,7 @@ class TestVisualization(BaseTest): visualization_annotation_association, visualization_rating_association, visualization_revision_factory, + visualization_user_share_association ): revision2 = visualization_revision_factory() persist(session, revision2) @@ -5155,6 +5179,7 @@ class TestVisualization(BaseTest): obj.tags.append(visualization_tag_association) obj.annotations.append(visualization_annotation_association) obj.ratings.append(visualization_rating_association) + obj.users_shared_with.append(visualization_user_share_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -5166,6 +5191,7 @@ class TestVisualization(BaseTest): assert stored_obj.ratings == [visualization_rating_association] # This doesn't test the average amount, just the mapping. assert stored_obj.average_rating == visualization_rating_association.rating + assert stored_obj.users_shared_with == [visualization_user_share_association] def test_average_rating(self, session, visualization, user, visualization_rating_association_factory): # Visualization has been expunged; to access its deferred properties, @@ -5433,6 +5459,7 @@ class TestWorkflowInvocation(BaseTest): workflow_request_step_state, workflow_request_to_input_dataset_association, workflow_request_to_input_dataset_collection_association, + workflow_invocation_output_value, ): subworkflow_invocation_assoc = workflow_invocation_to_subworkflow_invocation_association_factory() parent_workflow_invocation_assoc = workflow_invocation_to_subworkflow_invocation_association_factory() @@ -5450,6 +5477,7 @@ class TestWorkflowInvocation(BaseTest): obj.output_dataset_collections.append(workflow_invocation_output_dataset_collection_association) obj.output_datasets.append(workflow_invocation_output_dataset_association) obj.parent_workflow_invocation_association.append(parent_workflow_invocation_assoc) + obj.output_values.append(workflow_invocation_output_value) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -5466,6 +5494,7 @@ class TestWorkflowInvocation(BaseTest): assert stored_obj.output_dataset_collections == [workflow_invocation_output_dataset_collection_association] assert stored_obj.output_datasets == [workflow_invocation_output_dataset_association] assert stored_obj.parent_workflow_invocation_association == [parent_workflow_invocation_assoc] + assert stored_obj.output_values == [workflow_invocation_output_value] class TestWorkflowInvocationStep(BaseTest): @@ -6839,6 +6868,19 @@ def library_dataset_dataset_association_tag_association(model, session): yield from dbcleanup_wrapper(session, lddata) +@pytest.fixture +def library_dataset_dataset_info_association( + model, + session, + library_dataset_dataset_association, + form_definition, + form_values +): + instance = model.LibraryDatasetDatasetInfoAssociation( + library_dataset_dataset_association, form_definition, form_values) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def library_dataset_permission(model, session, library_dataset, role): ldp = model.LibraryDatasetPermissions('a', library_dataset, role) @@ -7097,6 +7139,12 @@ def visualization_tag_association(model, session): yield from dbcleanup_wrapper(session, vta) +@pytest.fixture +def visualization_user_share_association(model, session): + instance = model.VisualizationUserShareAssociation() + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def workflow(model, session): w = model.Workflow() From 0743363742c0ae2d1e66d59c4406eb3e3121a70c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 19:38:29 -0400 Subject: [PATCH 202/225] Cleanup: rename fixture local variables for consistency --- test/unit/model/test_model.py | 358 +++++++++++++++++----------------- 1 file changed, 179 insertions(+), 179 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 182a394cce9..42bfff37806 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -6420,14 +6420,14 @@ def api_keys(model, session): @pytest.fixture def cleanup_event(model, session): - ce = model.CleanupEvent() - yield from dbcleanup_wrapper(session, ce) + instance = model.CleanupEvent() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def cloud_authz(model, session, user, user_authnz_token): - ca = model.CloudAuthz(user.id, 'a', 'b', user_authnz_token.id, 'c') - yield from dbcleanup_wrapper(session, ca) + instance = model.CloudAuthz(user.id, 'a', 'b', user_authnz_token.id, 'c') + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6450,59 +6450,59 @@ def data_manager_job_association(model, session): @pytest.fixture def dataset(model, session): - d = model.Dataset() - yield from dbcleanup_wrapper(session, d) + instance = model.Dataset() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def dataset_collection(model, session): - dc = model.DatasetCollection(collection_type='a') - yield from dbcleanup_wrapper(session, dc) + instance = model.DatasetCollection(collection_type='a') + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def dataset_collection_element( model, session, dataset_collection, history_dataset_association): - dce = model.DatasetCollectionElement( + instance = model.DatasetCollectionElement( collection=dataset_collection, element=history_dataset_association) - yield from dbcleanup_wrapper(session, dce) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def dataset_hash(model, session): - dh = model.DatasetHash() - yield from dbcleanup_wrapper(session, dh) + instance = model.DatasetHash() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def dataset_permission(model, session, dataset): - d = model.DatasetPermissions('a', dataset) - yield from dbcleanup_wrapper(session, d) + instance = model.DatasetPermissions('a', dataset) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def dataset_source(model, session): - d = model.DatasetSource() - yield from dbcleanup_wrapper(session, d) + instance = model.DatasetSource() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def dataset_source_hash(model, session): - d = model.DatasetSourceHash() - yield from dbcleanup_wrapper(session, d) + instance = model.DatasetSourceHash() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def default_history_permissions(model, session, history, role): - dha = model.DefaultHistoryPermissions(history, 'a', role) - yield from dbcleanup_wrapper(session, dha) + instance = model.DefaultHistoryPermissions(history, 'a', role) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def default_quota_association(model, session, quota): type_ = model.DefaultQuotaAssociation.types.REGISTERED - dqa = model.DefaultQuotaAssociation(type_, quota) - yield from dbcleanup_wrapper(session, dqa) + instance = model.DefaultQuotaAssociation(type_, quota) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6513,98 +6513,98 @@ def default_user_permissions(model, session, user, role): @pytest.fixture def deferred_job(model, session): - dj = model.DeferredJob() - yield from dbcleanup_wrapper(session, dj) + instance = model.DeferredJob() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def dynamic_tool(model, session): - dt = model.DynamicTool() - yield from dbcleanup_wrapper(session, dt) + instance = model.DynamicTool() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def extended_metadata(model, session): - em = model.ExtendedMetadata(None) - yield from dbcleanup_wrapper(session, em) + instance = model.ExtendedMetadata(None) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def extended_metadata_index(model, session, extended_metadata): - emi = model.ExtendedMetadataIndex(extended_metadata, None, None) - yield from dbcleanup_wrapper(session, emi) + instance = model.ExtendedMetadataIndex(extended_metadata, None, None) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def form_definition(model, session, form_definition_current): - fd = model.FormDefinition(name='a', form_definition_current=form_definition_current) - yield from dbcleanup_wrapper(session, fd) + instance = model.FormDefinition(name='a', form_definition_current=form_definition_current) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def form_definition_current(model, session): - fdc = model.FormDefinitionCurrent() - yield from dbcleanup_wrapper(session, fdc) + instance = model.FormDefinitionCurrent() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def form_values(model, session): - fv = model.FormValues() - yield from dbcleanup_wrapper(session, fv) + instance = model.FormValues() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def galaxy_session(model, session, user): - s = model.GalaxySession() - yield from dbcleanup_wrapper(session, s) + instance = model.GalaxySession() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def galaxy_session_history_association(model, session, galaxy_session, history): - sha = model.GalaxySessionToHistoryAssociation(galaxy_session, history) - yield from dbcleanup_wrapper(session, sha) + instance = model.GalaxySessionToHistoryAssociation(galaxy_session, history) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def genome_index_tool_data(model, session): - gitd = model.GenomeIndexToolData() - yield from dbcleanup_wrapper(session, gitd) + instance = model.GenomeIndexToolData() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def group(model, session): - g = model.Group() - yield from dbcleanup_wrapper(session, g) + instance = model.Group() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def group_quota_association(model, session): - gqa = model.GroupQuotaAssociation(None, None) - yield from dbcleanup_wrapper(session, gqa) + instance = model.GroupQuotaAssociation(None, None) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def group_role_association(model, session): - gra = model.GroupRoleAssociation(None, None) - yield from dbcleanup_wrapper(session, gra) + instance = model.GroupRoleAssociation(None, None) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def history(model, session): - h = model.History() - yield from dbcleanup_wrapper(session, h) + instance = model.History() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def history_annotation_association(model, session): - haa = model.HistoryAnnotationAssociation() - yield from dbcleanup_wrapper(session, haa) + instance = model.HistoryAnnotationAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def history_dataset_association(model, session, dataset): - hda = model.HistoryDatasetAssociation(dataset=dataset) - yield from dbcleanup_wrapper(session, hda) + instance = model.HistoryDatasetAssociation(dataset=dataset) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6662,42 +6662,42 @@ def history_dataset_collection_tag_association(model, session): @pytest.fixture def history_rating_association(model, session, user, history): - hda = model.HistoryRatingAssociation(user, history) - yield from dbcleanup_wrapper(session, hda) + instance = model.HistoryRatingAssociation(user, history) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def history_tag_association(model, session): - hta = model.HistoryTagAssociation() - yield from dbcleanup_wrapper(session, hta) + instance = model.HistoryTagAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def history_user_share_association(model, session): - husa = model.HistoryUserShareAssociation() - yield from dbcleanup_wrapper(session, husa) + instance = model.HistoryUserShareAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def implicit_collection_jobs(model, session): - icj = model.ImplicitCollectionJobs(populated_state='new') - yield from dbcleanup_wrapper(session, icj) + instance = model.ImplicitCollectionJobs(populated_state='new') + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def implicit_collection_jobs_job_association(model, session): - icjja = model.ImplicitCollectionJobsJobAssociation() - icjja.order_index = 1 - yield from dbcleanup_wrapper(session, icjja) + instance = model.ImplicitCollectionJobsJobAssociation() + instance.order_index = 1 + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def implicitly_converted_dataset_association(model, session, history_dataset_association): - icda = model.ImplicitlyConvertedDatasetAssociation( + instance = model.ImplicitlyConvertedDatasetAssociation( dataset=history_dataset_association, parent=history_dataset_association, # using the same dataset; should work here. ) - yield from dbcleanup_wrapper(session, icda) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6714,8 +6714,8 @@ def interactive_tool_entry_point(model, session): @pytest.fixture def job(model, session): - j = model.Job() - yield from dbcleanup_wrapper(session, j) + instance = model.Job() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6726,8 +6726,8 @@ def job_container_association(model, session): @pytest.fixture def job_export_history_archive(model, session): - jeha = model.JobExportHistoryArchive() - yield from dbcleanup_wrapper(session, jeha) + instance = model.JobExportHistoryArchive() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6810,20 +6810,20 @@ def job_to_output_library_dataset_association(model, session, library_dataset_da @pytest.fixture def library(model, session): - lb = model.Library() - yield from dbcleanup_wrapper(session, lb) + instance = model.Library() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def library_folder(model, session): - lf = model.LibraryFolder() - yield from dbcleanup_wrapper(session, lf) + instance = model.LibraryFolder() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def library_dataset(model, session): - ld = model.LibraryDataset() - yield from dbcleanup_wrapper(session, ld) + instance = model.LibraryDataset() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6840,8 +6840,8 @@ def library_dataset_collection_annotation_association(model, session): @pytest.fixture def library_dataset_collection_association(model, session): - ldca = model.LibraryDatasetCollectionAssociation() - yield from dbcleanup_wrapper(session, ldca) + instance = model.LibraryDatasetCollectionAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6852,20 +6852,20 @@ def library_dataset_collection_rating_association(model, session): @pytest.fixture def library_dataset_collection_tag_association(model, session): - ldcta = model.LibraryDatasetCollectionTagAssociation() - yield from dbcleanup_wrapper(session, ldcta) + instance = model.LibraryDatasetCollectionTagAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def library_dataset_dataset_association(model, session): - ldda = model.LibraryDatasetDatasetAssociation() - yield from dbcleanup_wrapper(session, ldda) + instance = model.LibraryDatasetDatasetAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def library_dataset_dataset_association_tag_association(model, session): - lddata = model.LibraryDatasetDatasetAssociationTagAssociation() - yield from dbcleanup_wrapper(session, lddata) + instance = model.LibraryDatasetDatasetAssociationTagAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6883,82 +6883,82 @@ def library_dataset_dataset_info_association( @pytest.fixture def library_dataset_permission(model, session, library_dataset, role): - ldp = model.LibraryDatasetPermissions('a', library_dataset, role) - yield from dbcleanup_wrapper(session, ldp) + instance = model.LibraryDatasetPermissions('a', library_dataset, role) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def library_dataset_dataset_association_permission(model, session, library_dataset_dataset_association, role): - lddp = model.LibraryDatasetDatasetAssociationPermissions('a', library_dataset_dataset_association, role) - yield from dbcleanup_wrapper(session, lddp) + instance = model.LibraryDatasetDatasetAssociationPermissions('a', library_dataset_dataset_association, role) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def library_folder_info_association(model, session, library_folder, form_definition, form_values): - lfia = model.LibraryFolderInfoAssociation(library_folder, form_definition, form_values) - yield from dbcleanup_wrapper(session, lfia) + instance = model.LibraryFolderInfoAssociation(library_folder, form_definition, form_values) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def library_folder_permission(model, session, library_folder, role): - lfp = model.LibraryFolderPermissions('a', library_folder, role) - yield from dbcleanup_wrapper(session, lfp) + instance = model.LibraryFolderPermissions('a', library_folder, role) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def library_info_association(model, session, library, form_definition, form_values): - lia = model.LibraryInfoAssociation(library, form_definition, form_values) - yield from dbcleanup_wrapper(session, lia) + instance = model.LibraryInfoAssociation(library, form_definition, form_values) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def library_permission(model, session, library, role): - lp = model.LibraryPermissions('a', library, role) - yield from dbcleanup_wrapper(session, lp) + instance = model.LibraryPermissions('a', library, role) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def metadata_file(model, session): - mf = model.MetadataFile() - yield from dbcleanup_wrapper(session, mf) + instance = model.MetadataFile() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def page(model, session, user): - p = model.Page() - p.user = user - yield from dbcleanup_wrapper(session, p) + instance = model.Page() + instance.user = user + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def page_revision(model, session, page): - pr = model.PageRevision() - pr.page = page - yield from dbcleanup_wrapper(session, pr) + instance = model.PageRevision() + instance.page = page + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def page_annotation_association(model, session): - paa = model.PageAnnotationAssociation() - yield from dbcleanup_wrapper(session, paa) + instance = model.PageAnnotationAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def page_rating_association(model, session): - pra = model.PageRatingAssociation(None, None) - yield from dbcleanup_wrapper(session, pra) + instance = model.PageRatingAssociation(None, None) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def page_tag_association(model, session): - pta = model.PageTagAssociation() - yield from dbcleanup_wrapper(session, pta) + instance = model.PageTagAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def page_user_share_association(model, session): - pra = model.PageUserShareAssociation() - yield from dbcleanup_wrapper(session, pra) + instance = model.PageUserShareAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6971,8 +6971,8 @@ def password_reset_token(model, session, user): @pytest.fixture def post_job_action(model, session): - pja = model.PostJobAction('a') - yield from dbcleanup_wrapper(session, pja) + instance = model.PostJobAction('a') + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -6983,87 +6983,87 @@ def post_job_action_association(model, session, post_job_action, job): @pytest.fixture def quota(model, session): - q = model.Quota(get_random_string(), 'b') - yield from dbcleanup_wrapper(session, q) + instance = model.Quota(get_random_string(), 'b') + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def role(model, session): - r = model.Role() - yield from dbcleanup_wrapper(session, r) + instance = model.Role() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def stored_workflow(model, session, user): - w = model.StoredWorkflow() - w.user = user - yield from dbcleanup_wrapper(session, w) + instance = model.StoredWorkflow() + instance.user = user + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def stored_workflow_annotation_association(model, session): - haa = model.StoredWorkflowAnnotationAssociation() - yield from dbcleanup_wrapper(session, haa) + instance = model.StoredWorkflowAnnotationAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def stored_workflow_rating_association(model, session): - hda = model.StoredWorkflowRatingAssociation(None, None) - yield from dbcleanup_wrapper(session, hda) + instance = model.StoredWorkflowRatingAssociation(None, None) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def stored_workflow_tag_association(model, session): - swta = model.StoredWorkflowTagAssociation() - yield from dbcleanup_wrapper(session, swta) + instance = model.StoredWorkflowTagAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def stored_workflow_user_share_association(model, session): - husa = model.StoredWorkflowUserShareAssociation() - yield from dbcleanup_wrapper(session, husa) + instance = model.StoredWorkflowUserShareAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def tag(model, session): - t = model.Tag() - yield from dbcleanup_wrapper(session, t) + instance = model.Tag() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def task(model, session, job): - t = model.Task(job, 'a', 'b') - yield from dbcleanup_wrapper(session, t) + instance = model.Task(job, 'a', 'b') + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def task_metric_numeric(model, session): - tmn = model. TaskMetricNumeric('a', 'b', 9) - yield from dbcleanup_wrapper(session, tmn) + instance = model. TaskMetricNumeric('a', 'b', 9) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def task_metric_text(model, session): - tmt = model. TaskMetricText('a', 'b', 'c') - yield from dbcleanup_wrapper(session, tmt) + instance = model. TaskMetricText('a', 'b', 'c') + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def tool_tag_association(model, session): - tta = model.ToolTagAssociation() - yield from dbcleanup_wrapper(session, tta) + instance = model.ToolTagAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def transfer_job(model, session): - tj = model.TransferJob() - yield from dbcleanup_wrapper(session, tj) + instance = model.TransferJob() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def user(model, session): - u = model.User(email='test@example.com', password='password') - yield from dbcleanup_wrapper(session, u) + instance = model.User(email='test@example.com', password='password') + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -7080,8 +7080,8 @@ def user_address(model, session, user): @pytest.fixture def user_authnz_token(model, session, user): - t = model.UserAuthnzToken('a', 'b', 'c', 1, 'd', user) - yield from dbcleanup_wrapper(session, t) + instance = model.UserAuthnzToken('a', 'b', 'c', 1, 'd', user) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -7098,8 +7098,8 @@ def user_preference(model, session): @pytest.fixture def user_quota_association(model, session): - uqa = model.UserQuotaAssociation(None, None) - yield from dbcleanup_wrapper(session, uqa) + instance = model.UserQuotaAssociation(None, None) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -7110,33 +7110,33 @@ def user_role_association(model, session, user, role): @pytest.fixture def visualization(model, session, user): - v = model.Visualization() - v.user = user - yield from dbcleanup_wrapper(session, v) + instance = model.Visualization() + instance.user = user + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def visualization_annotation_association(model, session): - vaa = model.VisualizationAnnotationAssociation() - yield from dbcleanup_wrapper(session, vaa) + instance = model.VisualizationAnnotationAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def visualization_rating_association(model, session, user, visualization): - vaa = model.VisualizationRatingAssociation(user, visualization) - yield from dbcleanup_wrapper(session, vaa) + instance = model.VisualizationRatingAssociation(user, visualization) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def visualization_revision(model, session, visualization): - vr = model.VisualizationRevision(visualization=visualization) - yield from dbcleanup_wrapper(session, vr) + instance = model.VisualizationRevision(visualization=visualization) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def visualization_tag_association(model, session): - vta = model.VisualizationTagAssociation() - yield from dbcleanup_wrapper(session, vta) + instance = model.VisualizationTagAssociation() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -7147,15 +7147,15 @@ def visualization_user_share_association(model, session): @pytest.fixture def workflow(model, session): - w = model.Workflow() - yield from dbcleanup_wrapper(session, w) + instance = model.Workflow() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def workflow_invocation(model, session, workflow): - wi = model.WorkflowInvocation() - wi.workflow = workflow - yield from dbcleanup_wrapper(session, wi) + instance = model.WorkflowInvocation() + instance.workflow = workflow + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -7178,10 +7178,10 @@ def workflow_invocation_output_value(model, session): @pytest.fixture def workflow_invocation_step(model, session, workflow_invocation, workflow_step): - wis = model.WorkflowInvocationStep() - wis.workflow_invocation = workflow_invocation - wis.workflow_step = workflow_step - yield from dbcleanup_wrapper(session, wis) + instance = model.WorkflowInvocationStep() + instance.workflow_invocation = workflow_invocation + instance.workflow_step = workflow_step + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -7198,8 +7198,8 @@ def workflow_invocation_step_output_dataset_association(model, session): @pytest.fixture def workflow_output(model, session, workflow_step): - wo = model.WorkflowOutput(workflow_step) - yield from dbcleanup_wrapper(session, wo) + instance = model.WorkflowOutput(workflow_step) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture @@ -7234,27 +7234,27 @@ def workflow_request_to_input_dataset_collection_association(model, session): @pytest.fixture def workflow_step(model, session, workflow): - s = model.WorkflowStep() - s.workflow = workflow - yield from dbcleanup_wrapper(session, s) + instance = model.WorkflowStep() + instance.workflow = workflow + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def workflow_step_connection(model, session): - wsc = model.WorkflowStepConnection() - yield from dbcleanup_wrapper(session, wsc) + instance = model.WorkflowStepConnection() + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def workflow_step_input(model, session, workflow_step): - wsi = model.WorkflowStepInput(workflow_step) - yield from dbcleanup_wrapper(session, wsi) + instance = model.WorkflowStepInput(workflow_step) + yield from dbcleanup_wrapper(session, instance) @pytest.fixture def workflow_step_tag_association(model, session): - wsta = model.WorkflowStepTagAssociation() - yield from dbcleanup_wrapper(session, wsta) + instance = model.WorkflowStepTagAssociation() + yield from dbcleanup_wrapper(session, instance) # Fixtures yielding factory functions. From 01ab078e1f7a83d838081bb6d5cf0b3b11a0dfbb Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 21:56:46 -0400 Subject: [PATCH 203/225] Use uuid insted of random() for unique constraints Ensure unique values are used everywhere where there is a unique constraint. --- test/unit/model/test_model.py | 52 +++++++++++++++++------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 42bfff37806..d428874c778 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -42,7 +42,6 @@ TestFoo(BaseTest): # BaseTest is parent class assert stored_obj.user.id == user_id """ -import random from contextlib import contextmanager from datetime import datetime, timedelta from uuid import uuid4 @@ -77,7 +76,7 @@ class TestAPIKeys(BaseTest): assert cls_.__tablename__ == 'api_keys' def test_columns(self, session, cls_, user): - create_time, user_id, key = datetime.now(), user.id, get_random_string() + create_time, user_id, key = datetime.now(), user.id, get_unique_value() obj = cls_(user_id=user_id, key=key, create_time=create_time) with dbcleanup(session, obj) as obj_id: @@ -88,7 +87,7 @@ class TestAPIKeys(BaseTest): assert stored_obj.key == key def test_relationships(self, session, cls_, user): - obj = cls_(user_id=user.id, key=get_random_string()) + obj = cls_(user_id=user.id, key=get_unique_value()) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -1052,7 +1051,7 @@ class TestGalaxySession(BaseTest): remote_host = 'a' remote_addr = 'b' referer = 'c' - session_key = 'd' + session_key = get_unique_value() is_valid = True disk_usage = 9 last_action = update_time + timedelta(hours=1) @@ -1087,6 +1086,7 @@ class TestGalaxySession(BaseTest): def test_relationships(self, session, cls_, user, history, galaxy_session_history_association): obj = cls_(user=user, current_history=history) + obj.session_key = get_unique_value() obj.histories.append(galaxy_session_history_association) with dbcleanup(session, obj) as obj_id: @@ -3994,7 +3994,7 @@ class TestPasswordResetToken(BaseTest): assert cls_.__tablename__ == 'password_reset_token' def test_columns_and_relationships(self, session, cls_, user): - token = get_random_string() + token = get_unique_value() expiration_time = datetime.now() obj = cls_(user, token) obj.expiration_time = expiration_time @@ -4088,7 +4088,7 @@ class TestPSACode(BaseTest): assert has_unique_constraint(cls_.__table__, ('code', 'email')) def test_columns(self, session, cls_): - email, code = 'a', get_random_string() + email, code = 'a', get_unique_value() obj = cls_(email, code) with dbcleanup(session, obj) as obj_id: @@ -4139,7 +4139,7 @@ class TestQuota(BaseTest): assert cls_.__tablename__ == 'quota' def test_columns(self, session, cls_): - name, description, amount, operation = get_random_string(), 'b', 42, '+' + name, description, amount, operation = get_unique_value(), 'b', 42, '+' create_time = datetime.now() update_time = create_time + timedelta(hours=1) obj = cls_(name, description, amount, operation) @@ -4164,7 +4164,7 @@ class TestQuota(BaseTest): assoc_object.quota = obj getattr(obj, assoc_attribute).append(assoc_object) - obj = cls_(None, None, 1, '+') + obj = cls_(get_unique_value(), None, 1, '+') add_association(default_quota_association, 'default') add_association(group_quota_association, 'groups') add_association(user_quota_association, 'users') @@ -4182,7 +4182,7 @@ class TestRole(BaseTest): assert cls_.__tablename__ == 'role' def test_columns(self, session, cls_): - name, description, type_, deleted = get_random_string(), 'b', cls_.types.SYSTEM, True + name, description, type_, deleted = get_unique_value(), 'b', cls_.types.SYSTEM, True create_time = datetime.now() update_time = create_time + timedelta(hours=1) obj = cls_(name, description, type_, deleted) @@ -4211,7 +4211,7 @@ class TestRole(BaseTest): library_dataset_dataset_association_permission, user_role_association, ): - name, description, type_ = get_random_string(), 'b', cls_.types.SYSTEM + name, description, type_ = get_unique_value(), 'b', cls_.types.SYSTEM obj = cls_(name, description, type_) obj.dataset_actions.append(dataset_permission) obj.library_actions.append(library_permission) @@ -4759,8 +4759,8 @@ class TestUser(BaseTest): def test_columns(self, session, cls_, form_values): create_time = datetime.now() update_time = create_time + timedelta(hours=1) - email = get_random_string() - username = get_random_string() + email = get_unique_value() + username = get_unique_value() password = 'c' last_password_change = update_time external = True @@ -4834,8 +4834,8 @@ class TestUser(BaseTest): obj = cls_() - obj.email = get_random_string() - obj.username = get_random_string() + obj.email = get_unique_value() + obj.username = get_unique_value() obj.password = 'a' obj.values = form_values obj.addresses.append(user_address) @@ -4974,7 +4974,7 @@ class TestUserAuthnzToken(BaseTest): assert has_unique_constraint(cls_.__table__, ('provider', 'uid')) def test_columns(self, session, cls_, user): - provider, uid, extra_data, lifetime, assoc_type = get_random_string(), 'b', 'c', 1, 'd' + provider, uid, extra_data, lifetime, assoc_type = get_unique_value(), 'b', 'c', 1, 'd' obj = cls_(provider, uid, extra_data, lifetime, assoc_type, user) with dbcleanup(session, obj) as obj_id: @@ -4988,7 +4988,7 @@ class TestUserAuthnzToken(BaseTest): assert stored_obj.assoc_type == assoc_type def test_relationships(self, session, cls_, user, cloud_authz): - obj = cls_(get_random_string(), None, user=user) + obj = cls_(get_unique_value(), None, user=user) obj.cloudauthz.append(cloud_authz) with dbcleanup(session, obj) as obj_id: @@ -5331,7 +5331,7 @@ class TestWorkerProcess(BaseTest): assert has_unique_constraint(cls_.__table__, ('server_name', 'hostname')) def test_columns(self, session, cls_): - server_name, hostname = get_random_string(), 'a' + server_name, hostname = get_unique_value(), 'a' update_time = datetime.now() obj = cls_(server_name, hostname) obj.update_time = update_time @@ -6414,7 +6414,7 @@ def session(model): @pytest.fixture def api_keys(model, session): - instance = model.APIKeys() + instance = model.APIKeys(key=get_unique_value()) yield from dbcleanup_wrapper(session, instance) @@ -6555,7 +6555,7 @@ def form_values(model, session): @pytest.fixture def galaxy_session(model, session, user): - instance = model.GalaxySession() + instance = model.GalaxySession(session_key=get_unique_value()) yield from dbcleanup_wrapper(session, instance) @@ -6573,7 +6573,7 @@ def genome_index_tool_data(model, session): @pytest.fixture def group(model, session): - instance = model.Group() + instance = model.Group(name=get_unique_value()) yield from dbcleanup_wrapper(session, instance) @@ -6963,7 +6963,7 @@ def page_user_share_association(model, session): @pytest.fixture def password_reset_token(model, session, user): - token = get_random_string() + token = get_unique_value() instance = model.PasswordResetToken(user, token) where_clause = type(instance).token == token yield from dbcleanup_wrapper(session, instance, where_clause) @@ -6983,13 +6983,13 @@ def post_job_action_association(model, session, post_job_action, job): @pytest.fixture def quota(model, session): - instance = model.Quota(get_random_string(), 'b') + instance = model.Quota(get_unique_value(), 'b') yield from dbcleanup_wrapper(session, instance) @pytest.fixture def role(model, session): - instance = model.Role() + instance = model.Role(name=get_unique_value()) yield from dbcleanup_wrapper(session, instance) @@ -7062,7 +7062,7 @@ def transfer_job(model, session): @pytest.fixture def user(model, session): - instance = model.User(email='test@example.com', password='password') + instance = model.User(email=get_unique_value(), password='password') yield from dbcleanup_wrapper(session, instance) @@ -7499,6 +7499,6 @@ def has_index(table, fields): return True -def get_random_string(): +def get_unique_value(): """Generate unique values to accommodate unique constraints.""" - return str(random.random()) + return uuid4().hex From 9285eb19f8d9aeada81f9e7b85b7e3af3d78960f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 22:12:58 -0400 Subject: [PATCH 204/225] (Re)add tests for Group (deleted in a previous commit) --- test/unit/model/test_model.py | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index d428874c778..c8615447dad 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -1173,6 +1173,52 @@ class TestGenomeIndexToolData(BaseTest): assert stored_obj.user.id == user.id +class TestGroup(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'galaxy_group' + + def test_columns(self, session, cls_): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + name = get_unique_value() + deleted = True + + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.name = name + obj.deleted = deleted + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.deleted == deleted + + def test_relationships( + self, + session, + cls_, + group_quota_association, + group_role_association, + user_group_association, + ): + obj = cls_(name=get_unique_value()) + obj.quotas.append(group_quota_association) + obj.roles.append(group_role_association) + obj.users.append(user_group_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.quotas == [group_quota_association] + assert stored_obj.roles == [group_role_association] + assert stored_obj.users == [user_group_association] + + class TestGroupQuotaAssociation(BaseTest): def test_table(self, cls_): From 74ec7fec0b111d3e1c3511083d8368d953561202 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sun, 15 Aug 2021 22:27:03 -0400 Subject: [PATCH 205/225] Cleanup: review and remove TODOs from test module --- test/unit/model/test_model.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index c8615447dad..999a7125f09 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -441,8 +441,8 @@ class TestDataset(BaseTest): obj = cls_() obj.job = job obj.actions.append(dataset_permission) - obj.history_associations.append(history_dataset_association) # TODO: mappend via backref, map explicitly after HDA is mapped - obj.library_associations.append(library_dataset_dataset_association) # TODO: mappend via backref, map explicitly after LDDA is mapped + obj.history_associations.append(history_dataset_association) + obj.library_associations.append(library_dataset_dataset_association) obj.hashes.append(dataset_hash) obj.sources.append(dataset_source) @@ -1344,10 +1344,7 @@ class TestHistory(BaseTest): obj = cls_() obj.user = user obj.datasets.append(history_dataset_association) - - # TODO: this is mappend via backref: change to back_populates after mapping HDCA obj.dataset_collections.append(history_dataset_collection_association) - obj.exports.append(job_export_history_archive) obj.tags.append(history_tag_association) obj.annotations.append(history_annotation_association) @@ -2119,7 +2116,7 @@ class TestImplicitCollectionJobsJobAssociation(BaseTest): order_index = 1 obj = cls_() obj.implicit_collection_jobs = implicit_collection_jobs - session.add(job) # TODO review this after remapping Job (required to lazy-load attr) + session.add(job) obj.job = job obj.order_index = order_index @@ -2133,7 +2130,7 @@ class TestImplicitCollectionJobsJobAssociation(BaseTest): def test_relationships(self, session, cls_, implicit_collection_jobs, job): obj = cls_() obj.implicit_collection_jobs = implicit_collection_jobs - session.add(job) # TODO review this after remapping Job (required to lazy-load attr) + session.add(job) obj.job = job obj.order_index = 1 @@ -2528,8 +2525,7 @@ class TestJobContainerAssociation(BaseTest): created_time = datetime.now() modified_time = created_time + timedelta(hours=1) - session.add(job) # TODO review this after remapping Job (required to lazy-load `container` attr) - + session.add(job) obj = cls_(job, container_type, container_name, container_info) obj.created_time = created_time obj.modified_time = modified_time @@ -2545,7 +2541,7 @@ class TestJobContainerAssociation(BaseTest): assert stored_obj.modified_time == modified_time def test_relationships(self, session, cls_, job): - session.add(job) # TODO review this after remapping Job (required to lazy-load `container` attr) + session.add(job) obj = cls_(job, None, None, None) with dbcleanup(session, obj) as obj_id: @@ -3602,7 +3598,7 @@ class TestLibraryFolder(BaseTest): obj.info_association.append(library_folder_info_association) # There's no back reference, so dataset does not update folder; so we have to flush to the database - # via dbcleanup() context manager. TODO: ..but why is there no back reference? + # via dbcleanup() context manager. library_dataset.folder = obj with dbcleanup(session, obj) as obj_id, dbcleanup(session, library_dataset): @@ -5154,7 +5150,6 @@ class TestUserRoleAssociation(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.user.id == user.id assert stored_obj.role.id == role.id - # TODO non_private_roles should be defined on the User only (test to be added to TestUser) class TestVisualization(BaseTest): From 8983f2a5e3e1056872ae841ce26bd05434390aee Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 00:19:06 -0400 Subject: [PATCH 206/225] Sort tests alphabetically --- test/unit/model/test_model.py | 1286 ++++++++++++++++----------------- 1 file changed, 643 insertions(+), 643 deletions(-) diff --git a/test/unit/model/test_model.py b/test/unit/model/test_model.py index 999a7125f09..233fcd9b7e4 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_model.py @@ -127,23 +127,6 @@ class TestCleanupEventDatasetAssociation(BaseTest): assert stored_obj.dataset_id == dataset.id -class TestCleanupEventMetadataFileAssociation(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'cleanup_event_metadata_file_association' - - def test_columns(self, session, cls_, cleanup_event, metadata_file): - create_time = datetime.now() - obj = cls_(create_time=create_time, cleanup_event_id=cleanup_event.id, metadata_file_id=metadata_file.id) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.cleanup_event_id == cleanup_event.id - assert stored_obj.metadata_file_id == metadata_file.id - - class TestCleanupEventHistoryAssociation(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'cleanup_event_history_association' @@ -180,6 +163,26 @@ class TestCleanupEventHistoryDatasetAssociationAssociation(BaseTest): assert stored_obj.hda_id == history_dataset_association.id +class TestCleanupEventImplicitlyConvertedDatasetAssociationAssociation(BaseTest): + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_icda_association' + + def test_columns(self, session, cls_, cleanup_event, implicitly_converted_dataset_association): + create_time = datetime.now() + obj = cls_( + create_time=create_time, + cleanup_event_id=cleanup_event.id, + icda_id=implicitly_converted_dataset_association.id + ) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.icda_id == implicitly_converted_dataset_association.id + + class TestCleanupEventLibraryAssociation(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'cleanup_event_library_association' @@ -196,26 +199,6 @@ class TestCleanupEventLibraryAssociation(BaseTest): assert stored_obj.library_id == library.id -class TestCleanupEventLibraryFolderAssociation(BaseTest): - def test_table(self, cls_): - assert cls_.__tablename__ == 'cleanup_event_library_folder_association' - - def test_columns(self, session, cls_, cleanup_event, library_folder): - create_time = datetime.now() - obj = cls_( - create_time=create_time, - cleanup_event_id=cleanup_event.id, - library_folder_id=library_folder.id - ) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.cleanup_event_id == cleanup_event.id - assert stored_obj.library_folder_id == library_folder.id - - class TestCleanupEventLibraryDatasetAssociation(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'cleanup_event_library_dataset_association' @@ -256,16 +239,16 @@ class TestCleanupEventLibraryDatasetDatasetAssociationAssociation(BaseTest): assert stored_obj.ldda_id == library_dataset_dataset_association.id -class TestCleanupEventImplicitlyConvertedDatasetAssociationAssociation(BaseTest): +class TestCleanupEventLibraryFolderAssociation(BaseTest): def test_table(self, cls_): - assert cls_.__tablename__ == 'cleanup_event_icda_association' + assert cls_.__tablename__ == 'cleanup_event_library_folder_association' - def test_columns(self, session, cls_, cleanup_event, implicitly_converted_dataset_association): + def test_columns(self, session, cls_, cleanup_event, library_folder): create_time = datetime.now() obj = cls_( create_time=create_time, cleanup_event_id=cleanup_event.id, - icda_id=implicitly_converted_dataset_association.id + library_folder_id=library_folder.id ) with dbcleanup(session, obj) as obj_id: @@ -273,7 +256,24 @@ class TestCleanupEventImplicitlyConvertedDatasetAssociationAssociation(BaseTest) assert stored_obj.id == obj_id assert stored_obj.create_time == create_time assert stored_obj.cleanup_event_id == cleanup_event.id - assert stored_obj.icda_id == implicitly_converted_dataset_association.id + assert stored_obj.library_folder_id == library_folder.id + + +class TestCleanupEventMetadataFileAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'cleanup_event_metadata_file_association' + + def test_columns(self, session, cls_, cleanup_event, metadata_file): + create_time = datetime.now() + obj = cls_(create_time=create_time, cleanup_event_id=cleanup_event.id, metadata_file_id=metadata_file.id) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.cleanup_event_id == cleanup_event.id + assert stored_obj.metadata_file_id == metadata_file.id class TestCloudAuthz(BaseTest): @@ -608,6 +608,36 @@ class TestDatasetHash(BaseTest): assert stored_obj.dataset.id == dataset.id +class TestDatasetPermissions(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'dataset_permissions' + + def test_columns(self, session, cls_, dataset, role): + action = 'a' + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + obj = cls_(action, dataset, role) + obj.create_time = create_time + obj.update_time = update_time + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.update_time == update_time + assert stored_obj.action == action + assert stored_obj.dataset_id == dataset.id + assert stored_obj.role_id == role.id + + def test_relationships(self, session, cls_, dataset, role): + obj = cls_(None, dataset, role) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset == dataset + assert stored_obj.role == role + + class TestDatasetSource(BaseTest): def test_table(self, cls_): @@ -668,36 +698,6 @@ class TestDatasetSourceHash(BaseTest): assert stored_obj.source.id == dataset_source.id -class TestDatasetPermissions(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'dataset_permissions' - - def test_columns(self, session, cls_, dataset, role): - action = 'a' - create_time = datetime.now() - update_time = create_time + timedelta(hours=1) - obj = cls_(action, dataset, role) - obj.create_time = create_time - obj.update_time = update_time - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.update_time == update_time - assert stored_obj.action == action - assert stored_obj.dataset_id == dataset.id - assert stored_obj.role_id == role.id - - def test_relationships(self, session, cls_, dataset, role): - obj = cls_(None, dataset, role) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.dataset == dataset - assert stored_obj.role == role - - class TestDefaultHistoryPermissions(BaseTest): def test_table(self, cls_): @@ -1392,6 +1392,57 @@ class TestHistory(BaseTest): delete_from_database(session, to_cleanup) +class TestHistoryAnnotationAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_annotation_association' + assert has_index(cls_.__table__, ('annotation',)) + + def test_columns(self, session, cls_, history, user): + annotation = 'a' + obj = cls_() + obj.user = user + obj.history = history + obj.annotation = annotation + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id + assert stored_obj.annotation == annotation + + def test_relationships(self, session, cls_, history, user): + obj = cls_() + obj.user = user + obj.history = history + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history.id == history.id + assert stored_obj.user.id == user.id + + +class TestHistoryAudit(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_audit' + + def test_columns_and_relationships(self, session, cls_, history): + update_time = datetime.now() + obj = cls_(history, update_time) + + where_clause = and_(cls_.history_id == history.id, cls_.update_time == update_time) + + with dbcleanup(session, obj, where_clause): + stored_obj = get_stored_obj(session, cls_, where_clause=where_clause) + # test columns + assert stored_obj.history_id == history.id + assert stored_obj.update_time == update_time + # test relationships + assert stored_obj.history.id == history.id + + class TestHistoryDatasetAssociation(BaseTest): def test_table(self, cls_): @@ -1553,162 +1604,6 @@ class TestHistoryDatasetAssociation(BaseTest): delete_from_database(session, [copied_from_hda, copied_from_ldda, copied_to_ldda, icda, icpda]) -class TestHistoryDatasetAssociationHistory(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'history_dataset_association_history' - - def test_columns(self, session, cls_, history_dataset_association, extended_metadata): - name, update_time, version, extension, metadata = 'a', datetime.now(), 2, 'b', 'c' - obj = cls_( - history_dataset_association.id, - name, - None, - update_time, - version, - extension, - extended_metadata.id, - metadata - ) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.history_dataset_association_id == history_dataset_association.id - assert stored_obj.name == name - assert stored_obj.update_time == update_time - assert stored_obj.version == version - assert stored_obj.extension == extension - assert stored_obj.extended_metadata_id == extended_metadata.id - assert stored_obj._metadata == metadata - - -class TestHistoryDatasetAssociationSubset(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'history_dataset_association_subset' - - def test_columns( - self, - session, - cls_, - history_dataset_association, - history, - history_dataset_association_factory, - ): - hda_subset = history_dataset_association_factory() - persist(session, hda_subset) - - location = 'a' - obj = cls_(history_dataset_association, hda_subset, location) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.history_dataset_association_id == history_dataset_association.id - assert stored_obj.history_dataset_association_subset_id == hda_subset.id - assert stored_obj.location == location - - delete_from_database(session, [hda_subset]) - - def test_relationships( - self, - session, - cls_, - history_dataset_association, - history_dataset_association_factory, - ): - hda_subset = history_dataset_association_factory() - persist(session, hda_subset) - - obj = cls_(history_dataset_association, hda_subset, None) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.hda.id == history_dataset_association.id - assert stored_obj.subset.id == hda_subset.id - - delete_from_database(session, [hda_subset]) - - -class TestHistoryUserShareAssociation(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'history_user_share_association' - - def test_columns(self, session, cls_, history, user): - obj = cls_() - obj.history = history - obj.user = user - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.history_id == history.id - assert stored_obj.user_id == user.id - - def test_relationships(self, session, cls_, history, user): - obj = cls_() - obj.history = history - obj.user = user - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.history.id == history.id - assert stored_obj.user.id == user.id - - -class TestHistoryAudit(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'history_audit' - - def test_columns_and_relationships(self, session, cls_, history): - update_time = datetime.now() - obj = cls_(history, update_time) - - where_clause = and_(cls_.history_id == history.id, cls_.update_time == update_time) - - with dbcleanup(session, obj, where_clause): - stored_obj = get_stored_obj(session, cls_, where_clause=where_clause) - # test columns - assert stored_obj.history_id == history.id - assert stored_obj.update_time == update_time - # test relationships - assert stored_obj.history.id == history.id - - -class TestHistoryAnnotationAssociation(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'history_annotation_association' - assert has_index(cls_.__table__, ('annotation',)) - - def test_columns(self, session, cls_, history, user): - annotation = 'a' - obj = cls_() - obj.user = user - obj.history = history - obj.annotation = annotation - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.history_id == history.id - assert stored_obj.user_id == user.id - assert stored_obj.annotation == annotation - - def test_relationships(self, session, cls_, history, user): - obj = cls_() - obj.user = user - obj.history = history - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.history.id == history.id - assert stored_obj.user.id == user.id - - class TestHistoryDatasetAssociationAnnotationAssociation(BaseTest): def test_table(self, cls_): @@ -1769,6 +1664,36 @@ class TestHistoryDatasetAssociationDisplayAtAuthorization(BaseTest): assert stored_obj.user.id == user.id +class TestHistoryDatasetAssociationHistory(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_association_history' + + def test_columns(self, session, cls_, history_dataset_association, extended_metadata): + name, update_time, version, extension, metadata = 'a', datetime.now(), 2, 'b', 'c' + obj = cls_( + history_dataset_association.id, + name, + None, + update_time, + version, + extension, + extended_metadata.id, + metadata + ) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.name == name + assert stored_obj.update_time == update_time + assert stored_obj.version == version + assert stored_obj.extension == extension + assert stored_obj.extended_metadata_id == extended_metadata.id + assert stored_obj._metadata == metadata + + class TestHistoryDatasetAssociationRatingAssociation(BaseTest): def test_table(self, cls_): @@ -1793,6 +1718,54 @@ class TestHistoryDatasetAssociationRatingAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestHistoryDatasetAssociationSubset(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_dataset_association_subset' + + def test_columns( + self, + session, + cls_, + history_dataset_association, + history, + history_dataset_association_factory, + ): + hda_subset = history_dataset_association_factory() + persist(session, hda_subset) + + location = 'a' + obj = cls_(history_dataset_association, hda_subset, location) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_dataset_association_id == history_dataset_association.id + assert stored_obj.history_dataset_association_subset_id == hda_subset.id + assert stored_obj.location == location + + delete_from_database(session, [hda_subset]) + + def test_relationships( + self, + session, + cls_, + history_dataset_association, + history_dataset_association_factory, + ): + hda_subset = history_dataset_association_factory() + persist(session, hda_subset) + + obj = cls_(history_dataset_association, hda_subset, None) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.hda.id == history_dataset_association.id + assert stored_obj.subset.id == hda_subset.id + + delete_from_database(session, [hda_subset]) + + class TestHistoryDatasetAssociationTagAssociation(BaseTest): def test_table(self, cls_): @@ -2072,6 +2045,33 @@ class TestHistoryTagAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestHistoryUserShareAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'history_user_share_association' + + def test_columns(self, session, cls_, history, user): + obj = cls_() + obj.history = history + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.history_id == history.id + assert stored_obj.user_id == user.id + + def test_relationships(self, session, cls_, history, user): + obj = cls_() + obj.history = history + obj.user = user + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.history.id == history.id + assert stored_obj.user.id == user.id + + class TestImplicitCollectionJobs(BaseTest): def test_table(self, cls_): @@ -2803,6 +2803,35 @@ class TestJobToImplicitOutputDatasetCollectionAssociation(BaseTest): assert stored_obj.dataset_collection.id == dataset_collection.id +class TestJobToInputDatasetAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'job_to_input_dataset' + + def test_columns(self, session, cls_, history_dataset_association, job): + name, dataset_version = 'a', 9 + obj = cls_(name, history_dataset_association) + obj.job = job + obj.dataset_version = dataset_version + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.job_id == job.id + assert stored_obj.dataset_id == history_dataset_association.id + assert stored_obj.dataset_version == dataset_version + assert stored_obj.name == name + + def test_relationships(self, session, cls_, history_dataset_association, job): + obj = cls_(None, history_dataset_association) + obj.job = job + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.job.id == job.id + assert stored_obj.dataset.id == history_dataset_association.id + + class TestJobToInputDatasetCollectionAssociation(BaseTest): def test_table(self, cls_): @@ -2857,35 +2886,6 @@ class TestJobToInputDatasetCollectionElementAssociation(BaseTest): assert stored_obj.dataset_collection_element.id == dataset_collection_element.id -class TestJobToInputDatasetAssociation(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'job_to_input_dataset' - - def test_columns(self, session, cls_, history_dataset_association, job): - name, dataset_version = 'a', 9 - obj = cls_(name, history_dataset_association) - obj.job = job - obj.dataset_version = dataset_version - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.job_id == job.id - assert stored_obj.dataset_id == history_dataset_association.id - assert stored_obj.dataset_version == dataset_version - assert stored_obj.name == name - - def test_relationships(self, session, cls_, history_dataset_association, job): - obj = cls_(None, history_dataset_association) - obj.job = job - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.job.id == job.id - assert stored_obj.dataset.id == history_dataset_association.id - - class TestJobToInputLibraryDatasetAssociation(BaseTest): def test_table(self, cls_): @@ -2967,116 +2967,6 @@ class TestJobToOutputDatasetCollectionAssociation(BaseTest): assert stored_obj.dataset_collection_instance.id == history_dataset_collection_association.id -class TestLibraryDataset(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'library_dataset' - - def test_columns(self, session, cls_, library_dataset_dataset_association, library_folder): - order_id = 9 - create_time = datetime.now() - update_time = create_time + timedelta(hours=1) - name = 'a' - info = 'b' - deleted = False - purged = False - - obj = cls_() - obj.folder = library_folder - obj.order_id = order_id - obj.create_time = create_time - obj.update_time = update_time - obj.name = name - obj.info = info - obj.deleted = deleted - obj.purged = purged - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.folder_id == library_folder.id - assert stored_obj.order_id == order_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.name == name - assert stored_obj.info == info - assert stored_obj.deleted == deleted - assert stored_obj.purged == purged - - def test_relationships( - self, - session, - cls_, - library_dataset_dataset_association, - library_folder, - library_dataset_permission, - library_dataset_dataset_association_factory, - ): - obj = cls_() - obj.folder = library_folder - obj.library_dataset_dataset_association = library_dataset_dataset_association - obj.actions.append(library_dataset_permission) - - ldda = library_dataset_dataset_association_factory() - ldda.library_dataset = obj - obj.actions.append(library_dataset_permission) - - with dbcleanup(session, obj) as obj_id, dbcleanup(session, ldda): - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id - assert stored_obj.folder.id == library_folder.id - assert stored_obj.expired_datasets[0].id == ldda.id - assert stored_obj.actions == [library_dataset_permission] - - -class TestLibraryDatasetCollectionAssociation(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'library_dataset_collection_association' - - def test_columns(self, session, cls_, dataset_collection, library_folder): - name, deleted = 'a', True - obj = cls_() - obj.collection = dataset_collection - obj.folder = library_folder - obj.name = name - obj.deleted = deleted - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.collection_id == dataset_collection.id - assert stored_obj.folder_id == library_folder.id - assert stored_obj.name == name - assert stored_obj.deleted == deleted - - def test_relationships( - self, - session, - cls_, - dataset_collection, - library_folder, - library_dataset_collection_annotation_association, - library_dataset_collection_rating_association, - library_dataset_collection_tag_association, - ): - - obj = cls_() - obj.collection = dataset_collection - obj.folder = library_folder - obj.tags.append(library_dataset_collection_tag_association) - obj.annotations.append(library_dataset_collection_annotation_association) - obj.ratings.append(library_dataset_collection_rating_association) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.collection.id == dataset_collection.id - assert stored_obj.folder.id == library_folder.id - assert stored_obj.tags == [library_dataset_collection_tag_association] - assert stored_obj.annotations == [library_dataset_collection_annotation_association] - assert stored_obj.ratings == [library_dataset_collection_rating_association] - - class TestJobToOutputLibraryDatasetAssociation(BaseTest): def test_table(self, cls_): @@ -3151,6 +3041,68 @@ class TestLibrary(BaseTest): assert stored_obj.info_association == [library_info_association] +class TestLibraryDataset(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset' + + def test_columns(self, session, cls_, library_dataset_dataset_association, library_folder): + order_id = 9 + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + name = 'a' + info = 'b' + deleted = False + purged = False + + obj = cls_() + obj.folder = library_folder + obj.order_id = order_id + obj.create_time = create_time + obj.update_time = update_time + obj.name = name + obj.info = info + obj.deleted = deleted + obj.purged = purged + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.folder_id == library_folder.id + assert stored_obj.order_id == order_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.name == name + assert stored_obj.info == info + assert stored_obj.deleted == deleted + assert stored_obj.purged == purged + + def test_relationships( + self, + session, + cls_, + library_dataset_dataset_association, + library_folder, + library_dataset_permission, + library_dataset_dataset_association_factory, + ): + obj = cls_() + obj.folder = library_folder + obj.library_dataset_dataset_association = library_dataset_dataset_association + obj.actions.append(library_dataset_permission) + + ldda = library_dataset_dataset_association_factory() + ldda.library_dataset = obj + obj.actions.append(library_dataset_permission) + + with dbcleanup(session, obj) as obj_id, dbcleanup(session, ldda): + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert stored_obj.folder.id == library_folder.id + assert stored_obj.expired_datasets[0].id == ldda.id + assert stored_obj.actions == [library_dataset_permission] + + class TestLibraryDatasetCollectionAnnotationAssociation(BaseTest): def test_table(self, cls_): @@ -3181,6 +3133,79 @@ class TestLibraryDatasetCollectionAnnotationAssociation(BaseTest): assert stored_obj.user.id == user.id +class TestLibraryDatasetCollectionAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset_collection_association' + + def test_columns(self, session, cls_, dataset_collection, library_folder): + name, deleted = 'a', True + obj = cls_() + obj.collection = dataset_collection + obj.folder = library_folder + obj.name = name + obj.deleted = deleted + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.collection_id == dataset_collection.id + assert stored_obj.folder_id == library_folder.id + assert stored_obj.name == name + assert stored_obj.deleted == deleted + + def test_relationships( + self, + session, + cls_, + dataset_collection, + library_folder, + library_dataset_collection_annotation_association, + library_dataset_collection_rating_association, + library_dataset_collection_tag_association, + ): + + obj = cls_() + obj.collection = dataset_collection + obj.folder = library_folder + obj.tags.append(library_dataset_collection_tag_association) + obj.annotations.append(library_dataset_collection_annotation_association) + obj.ratings.append(library_dataset_collection_rating_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.collection.id == dataset_collection.id + assert stored_obj.folder.id == library_folder.id + assert stored_obj.tags == [library_dataset_collection_tag_association] + assert stored_obj.annotations == [library_dataset_collection_annotation_association] + assert stored_obj.ratings == [library_dataset_collection_rating_association] + + +class TestLibraryDatasetCollectionRatingAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'library_dataset_collection_rating_association' + + def test_columns(self, session, cls_, library_dataset_collection_association, user): + rating = 9 + obj = cls_(user, library_dataset_collection_association, rating) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id + assert stored_obj.user_id == user.id + assert stored_obj.rating == rating + + def test_relationships(self, session, cls_, library_dataset_collection_association, user): + obj = cls_(user, library_dataset_collection_association, 1) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.dataset_collection.id == library_dataset_collection_association.id + assert stored_obj.user.id == user.id + + class TestLibraryDatasetCollectionTagAssociation(BaseTest): def test_table(self, cls_): @@ -3213,31 +3238,6 @@ class TestLibraryDatasetCollectionTagAssociation(BaseTest): assert stored_obj.user.id == user.id -class TestLibraryDatasetCollectionRatingAssociation(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'library_dataset_collection_rating_association' - - def test_columns(self, session, cls_, library_dataset_collection_association, user): - rating = 9 - obj = cls_(user, library_dataset_collection_association, rating) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id - assert stored_obj.user_id == user.id - assert stored_obj.rating == rating - - def test_relationships(self, session, cls_, library_dataset_collection_association, user): - obj = cls_(user, library_dataset_collection_association, 1) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.dataset_collection.id == library_dataset_collection_association.id - assert stored_obj.user.id == user.id - - class TestLibraryDatasetDatasetAssociation(BaseTest): def test_table(self, cls_): @@ -3796,6 +3796,78 @@ class TestMetadataFile(BaseTest): assert stored_obj.library_dataset.id == library_dataset_dataset_association.id +class TestPSAAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'psa_association' + + def test_columns(self, session, cls_): + server_url, handle, secret, issued, lifetime, assoc_type = 'a', 'b', 'c', 1, 2, 'd' + obj = cls_(server_url, handle, secret, issued, lifetime, assoc_type) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.server_url == server_url + assert stored_obj.handle == handle + assert stored_obj.secret == secret + assert stored_obj.issued == issued + assert stored_obj.lifetime == lifetime + assert stored_obj.assoc_type == assoc_type + + +class TestPSACode(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'psa_code' + assert has_unique_constraint(cls_.__table__, ('code', 'email')) + + def test_columns(self, session, cls_): + email, code = 'a', get_unique_value() + obj = cls_(email, code) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.email == email + assert stored_obj.code == code + + +class TestPSANonce(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'psa_nonce' + + def test_columns(self, session, cls_): + server_url, timestamp, salt = 'a', 1, 'b' + obj = cls_(server_url, timestamp, salt) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.server_url + assert stored_obj.timestamp == timestamp + assert stored_obj.salt == salt + + +class TestPSAPartial(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'psa_partial' + + def test_columns(self, session, cls_): + token, data, next_step, backend = 'a', 'b', 1, 'c' + obj = cls_(token, data, next_step, backend) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.token == token + assert stored_obj.data == data + assert stored_obj.next_step == next_step + assert stored_obj.backend == backend + + class TestPage(BaseTest): def test_table(self, cls_): @@ -4103,78 +4175,6 @@ class TestPostJobActionAssociation(BaseTest): assert stored_obj.post_job_action.id == post_job_action.id -class TestPSAAssociation(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'psa_association' - - def test_columns(self, session, cls_): - server_url, handle, secret, issued, lifetime, assoc_type = 'a', 'b', 'c', 1, 2, 'd' - obj = cls_(server_url, handle, secret, issued, lifetime, assoc_type) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.server_url == server_url - assert stored_obj.handle == handle - assert stored_obj.secret == secret - assert stored_obj.issued == issued - assert stored_obj.lifetime == lifetime - assert stored_obj.assoc_type == assoc_type - - -class TestPSACode(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'psa_code' - assert has_unique_constraint(cls_.__table__, ('code', 'email')) - - def test_columns(self, session, cls_): - email, code = 'a', get_unique_value() - obj = cls_(email, code) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.email == email - assert stored_obj.code == code - - -class TestPSANonce(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'psa_nonce' - - def test_columns(self, session, cls_): - server_url, timestamp, salt = 'a', 1, 'b' - obj = cls_(server_url, timestamp, salt) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.server_url - assert stored_obj.timestamp == timestamp - assert stored_obj.salt == salt - - -class TestPSAPartial(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'psa_partial' - - def test_columns(self, session, cls_): - token, data, next_step, backend = 'a', 'b', 1, 'c' - obj = cls_(token, data, next_step, backend) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.token == token - assert stored_obj.data == data - assert stored_obj.next_step == next_step - assert stored_obj.backend == backend - - class TestQuota(BaseTest): def test_table(self, cls_): @@ -5338,6 +5338,38 @@ class TestVisualizationRevision(BaseTest): assert stored_obj.visualization.id == visualization.id +class TestVisualizationTagAssociation(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'visualization_tag_association' + + def test_columns(self, session, cls_, visualization, tag, user): + user_tname, value, user_value = 'a', 'b', 'c' + obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) + obj.user_value = user_value + obj.visualization = visualization + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.visualization_id == visualization.id + assert stored_obj.tag_id == tag.id + assert stored_obj.user_id == user.id + assert stored_obj.user_tname == user_tname + assert stored_obj.value == value + assert stored_obj.user_value == user_value + + def test_relationships(self, session, cls_, visualization, tag, user): + obj = cls_(user=user, tag=tag) + obj.visualization = visualization + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.visualization.id == visualization.id + assert stored_obj.tag.id == tag.id + assert stored_obj.user.id == user.id + + class TestVisualizationUserShareAssociation(BaseTest): def test_table(self, cls_): @@ -5538,126 +5570,6 @@ class TestWorkflowInvocation(BaseTest): assert stored_obj.output_values == [workflow_invocation_output_value] -class TestWorkflowInvocationStep(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'workflow_invocation_step' - - def test_columns( - self, - session, - cls_, - workflow_invocation, - workflow_step, - job, - implicit_collection_jobs, - ): - create_time = datetime.now() - update_time = create_time + timedelta(hours=1) - state, action = 'a', 'b' - - session.add(job) # needed for a lazy load of an attr - session.add(implicit_collection_jobs) # needed for a lazy load of an attr - - obj = cls_() - obj.create_time = create_time - obj.update_time = update_time - obj.workflow_invocation = workflow_invocation - obj.workflow_step = workflow_step - obj.state = state - obj.job = job - obj.implicit_collection_jobs = implicit_collection_jobs - obj.action = action - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.update_time == update_time - assert stored_obj.workflow_invocation_id == workflow_invocation.id - assert stored_obj.workflow_step_id == workflow_step.id - assert stored_obj.state == state - assert stored_obj.job_id == job.id - assert stored_obj.implicit_collection_jobs_id == implicit_collection_jobs.id - assert stored_obj.action == action - - def test_relationships( - self, - session, - cls_, - workflow_invocation, - workflow_step, - job, - implicit_collection_jobs, - workflow_invocation_step_output_dataset_collection_association, - workflow_invocation_step_output_dataset_association, - workflow_invocation_output_value, - ): - session.add(job) # needed for a lazy load of an attr - session.add(implicit_collection_jobs) # needed for a lazy load of an attr - - # setup workflow_invocation_output_value to test the output_value attribute - output_value = workflow_invocation_output_value - output_value.workflow_invocation = workflow_invocation - output_value.workflow_step = workflow_step - persist(session, output_value) - - obj = cls_() - obj.workflow_invocation = workflow_invocation - obj.workflow_step = workflow_step - obj.job = job - obj.implicit_collection_jobs = implicit_collection_jobs - obj.output_dataset_collections.append(workflow_invocation_step_output_dataset_collection_association) - obj.output_datasets.append(workflow_invocation_step_output_dataset_association) - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.workflow_invocation.id == workflow_invocation.id - assert stored_obj.workflow_step.id == workflow_step.id - assert stored_obj.job.id == job.id - assert stored_obj.implicit_collection_jobs.id == implicit_collection_jobs.id - assert stored_obj.output_dataset_collections == [workflow_invocation_step_output_dataset_collection_association] - assert stored_obj.output_datasets == [workflow_invocation_step_output_dataset_association] - assert stored_obj.output_value.id == workflow_invocation_output_value.id - - def test_subworkflow_invocation_attribute( - self, - session, - cls_, - workflow_step, - workflow_invocation_to_subworkflow_invocation_association_factory, - workflow_invocation_factory, - ): - # use defaults to create 2 workflows - workflow_invocation1 = workflow_invocation_factory() - workflow_invocation2 = workflow_invocation_factory() # this is the subworkflow invocation - - # store to retrieve object ids - persist(session, workflow_invocation1) - persist(session, workflow_invocation2) - persist(session, workflow_step) - - # setup assoc object and store - assoc = workflow_invocation_to_subworkflow_invocation_association_factory() - assoc.workflow_invocation_id = workflow_invocation1.id - assoc.subworkflow_invocation_id = workflow_invocation2.id - assoc.workflow_step_id = workflow_step.id - persist(session, assoc) - - # setup main object under test - obj = cls_() - obj.workflow_invocation = workflow_invocation1 - obj.workflow_step = workflow_step - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.subworkflow_invocation_id == workflow_invocation2.id - - # finish cleanup - persisted = [workflow_invocation1, workflow_invocation2, workflow_step, assoc] - delete_from_database(session, persisted) - - class TestWorkflowInvocationOutputDatasetAssociation(BaseTest): def test_table(self, cls_): @@ -5817,6 +5729,126 @@ class TestWorkflowInvocationOutputValue(BaseTest): delete_from_database(session, [workflow_invocation_step]) +class TestWorkflowInvocationStep(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_invocation_step' + + def test_columns( + self, + session, + cls_, + workflow_invocation, + workflow_step, + job, + implicit_collection_jobs, + ): + create_time = datetime.now() + update_time = create_time + timedelta(hours=1) + state, action = 'a', 'b' + + session.add(job) # needed for a lazy load of an attr + session.add(implicit_collection_jobs) # needed for a lazy load of an attr + + obj = cls_() + obj.create_time = create_time + obj.update_time = update_time + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.state = state + obj.job = job + obj.implicit_collection_jobs = implicit_collection_jobs + obj.action = action + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.create_time == create_time + assert stored_obj.update_time == update_time + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.workflow_step_id == workflow_step.id + assert stored_obj.state == state + assert stored_obj.job_id == job.id + assert stored_obj.implicit_collection_jobs_id == implicit_collection_jobs.id + assert stored_obj.action == action + + def test_relationships( + self, + session, + cls_, + workflow_invocation, + workflow_step, + job, + implicit_collection_jobs, + workflow_invocation_step_output_dataset_collection_association, + workflow_invocation_step_output_dataset_association, + workflow_invocation_output_value, + ): + session.add(job) # needed for a lazy load of an attr + session.add(implicit_collection_jobs) # needed for a lazy load of an attr + + # setup workflow_invocation_output_value to test the output_value attribute + output_value = workflow_invocation_output_value + output_value.workflow_invocation = workflow_invocation + output_value.workflow_step = workflow_step + persist(session, output_value) + + obj = cls_() + obj.workflow_invocation = workflow_invocation + obj.workflow_step = workflow_step + obj.job = job + obj.implicit_collection_jobs = implicit_collection_jobs + obj.output_dataset_collections.append(workflow_invocation_step_output_dataset_collection_association) + obj.output_datasets.append(workflow_invocation_step_output_dataset_association) + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id + assert stored_obj.workflow_step.id == workflow_step.id + assert stored_obj.job.id == job.id + assert stored_obj.implicit_collection_jobs.id == implicit_collection_jobs.id + assert stored_obj.output_dataset_collections == [workflow_invocation_step_output_dataset_collection_association] + assert stored_obj.output_datasets == [workflow_invocation_step_output_dataset_association] + assert stored_obj.output_value.id == workflow_invocation_output_value.id + + def test_subworkflow_invocation_attribute( + self, + session, + cls_, + workflow_step, + workflow_invocation_to_subworkflow_invocation_association_factory, + workflow_invocation_factory, + ): + # use defaults to create 2 workflows + workflow_invocation1 = workflow_invocation_factory() + workflow_invocation2 = workflow_invocation_factory() # this is the subworkflow invocation + + # store to retrieve object ids + persist(session, workflow_invocation1) + persist(session, workflow_invocation2) + persist(session, workflow_step) + + # setup assoc object and store + assoc = workflow_invocation_to_subworkflow_invocation_association_factory() + assoc.workflow_invocation_id = workflow_invocation1.id + assoc.subworkflow_invocation_id = workflow_invocation2.id + assoc.workflow_step_id = workflow_step.id + persist(session, assoc) + + # setup main object under test + obj = cls_() + obj.workflow_invocation = workflow_invocation1 + obj.workflow_step = workflow_step + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.subworkflow_invocation_id == workflow_invocation2.id + + # finish cleanup + persisted = [workflow_invocation1, workflow_invocation2, workflow_step, assoc] + delete_from_database(session, persisted) + + class TestWorkflowInvocationStepOutputDatasetAssociation(BaseTest): def test_table(self, cls_): @@ -5973,6 +6005,33 @@ class TestWorkflowOutput(BaseTest): assert stored_obj.workflow_step.id == workflow_step.id +class TestWorkflowRequestInputParameter(BaseTest): + + def test_table(self, cls_): + assert cls_.__tablename__ == 'workflow_request_input_parameters' + + def test_columns(self, session, cls_, workflow_invocation): + name, value, type = 'a', 'b', 'c' + obj = cls_(name, value, type) + obj.workflow_invocation = workflow_invocation + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.id == obj_id + assert stored_obj.workflow_invocation_id == workflow_invocation.id + assert stored_obj.name == name + assert stored_obj.value == value + assert stored_obj.type == type + + def test_relationships(self, session, cls_, workflow_invocation): + obj = cls_() + obj.workflow_invocation = workflow_invocation + + with dbcleanup(session, obj) as obj_id: + stored_obj = get_stored_obj(session, cls_, obj_id) + assert stored_obj.workflow_invocation.id == workflow_invocation.id + + class TestWorkflowRequestInputStepParameter(BaseTest): def test_table(self, cls_): @@ -6377,65 +6436,6 @@ class TestWorkflowStepTagAssociation(BaseTest): assert stored_obj.user.id == user.id -class TestVisualizationTagAssociation(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'visualization_tag_association' - - def test_columns(self, session, cls_, visualization, tag, user): - user_tname, value, user_value = 'a', 'b', 'c' - obj = cls_(user=user, tag=tag, user_tname=user_tname, value=value) - obj.user_value = user_value - obj.visualization = visualization - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.visualization_id == visualization.id - assert stored_obj.tag_id == tag.id - assert stored_obj.user_id == user.id - assert stored_obj.user_tname == user_tname - assert stored_obj.value == value - assert stored_obj.user_value == user_value - - def test_relationships(self, session, cls_, visualization, tag, user): - obj = cls_(user=user, tag=tag) - obj.visualization = visualization - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.visualization.id == visualization.id - assert stored_obj.tag.id == tag.id - assert stored_obj.user.id == user.id - - -class TestWorkflowRequestInputParameter(BaseTest): - - def test_table(self, cls_): - assert cls_.__tablename__ == 'workflow_request_input_parameters' - - def test_columns(self, session, cls_, workflow_invocation): - name, value, type = 'a', 'b', 'c' - obj = cls_(name, value, type) - obj.workflow_invocation = workflow_invocation - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.id == obj_id - assert stored_obj.workflow_invocation_id == workflow_invocation.id - assert stored_obj.name == name - assert stored_obj.value == value - assert stored_obj.type == type - - def test_relationships(self, session, cls_, workflow_invocation): - obj = cls_() - obj.workflow_invocation = workflow_invocation - - with dbcleanup(session, obj) as obj_id: - stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.workflow_invocation.id == workflow_invocation.id - - # Misc. helper fixtures. @pytest.fixture(scope='module') From e1662f81582684a1876028a9fda9a84a856fa3ec Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 00:20:56 -0400 Subject: [PATCH 207/225] Rename test module to test_mapping Because it tests the mapping of model classes to SQLAlchemy's column- and relationship-mapped attributes. We do not test model logic in the module. (also, fix fixture alphabetical sorting) --- _fix | 135 ++++++++++++++++++ _fix-sort | 135 ++++++++++++++++++ .../model/{test_model.py => test_mapping.py} | 82 +++++------ 3 files changed, 311 insertions(+), 41 deletions(-) create mode 100644 _fix create mode 100644 _fix-sort rename test/unit/model/{test_model.py => test_mapping.py} (100%) diff --git a/_fix b/_fix new file mode 100644 index 00000000000..f89eba342f7 --- /dev/null +++ b/_fix @@ -0,0 +1,135 @@ +def api_keys(model, session): +def cleanup_event(model, session): +def cloud_authz(model, session, user, user_authnz_token): +def custos_authnz_token(model, session, user, user_authnz_token): +def data_manager_history_association(model, session): +def data_manager_job_association(model, session): +def dataset(model, session): +def dataset_collection(model, session): +def dataset_collection_element( +def dataset_hash(model, session): +def dataset_permission(model, session, dataset): +def dataset_source(model, session): +def dataset_source_hash(model, session): +def default_history_permissions(model, session, history, role): +def default_quota_association(model, session, quota): +def default_user_permissions(model, session, user, role): +def deferred_job(model, session): +def dynamic_tool(model, session): +def extended_metadata(model, session): +def extended_metadata_index(model, session, extended_metadata): +def form_definition(model, session, form_definition_current): +def form_definition_current(model, session): +def form_values(model, session): +def galaxy_session(model, session, user): +def galaxy_session_history_association(model, session, galaxy_session, history): +def genome_index_tool_data(model, session): +def group(model, session): +def group_quota_association(model, session): +def group_role_association(model, session): +def history(model, session): +def history_annotation_association(model, session): +def history_dataset_annotation_association(model, session): +def history_dataset_association(model, session, dataset): +def history_dataset_association_annotation_association(model, session): +def history_dataset_association_rating_association(model, session): +def history_dataset_association_tag_association(model, session): +def history_dataset_collection_annotation_association(model, session): +def history_dataset_collection_association(model, session): +def history_dataset_collection_rating_association( +def history_dataset_collection_tag_association(model, session): +def history_rating_association(model, session, user, history): +def history_tag_association(model, session): +def history_user_share_association(model, session): +def implicit_collection_jobs(model, session): +def implicit_collection_jobs_job_association(model, session): +def implicitly_converted_dataset_association(model, session, history_dataset_association): +def implicitly_created_dataset_collection_input(model, session, history_dataset_collection_association): +def interactive_tool_entry_point(model, session): +def job(model, session): +def job_container_association(model, session): +def job_export_history_archive(model, session): +def job_external_output_metadata(model, session, job, history_dataset_association): +def job_metric_numeric(model, session): +def job_metric_text(model, session): +def job_parameter(model, session): +def job_state_history(model, session, job): +def job_to_implicit_output_dataset_collection_association(model, session, dataset_collection): +def job_to_input_dataset_association(model, session, history_dataset_association): +def job_to_input_dataset_collection_association(model, session, history_dataset_collection_association): +def job_to_input_dataset_collection_element_association(model, session, dataset_collection_element): +def job_to_input_library_dataset_association(model, session, library_dataset_dataset_association): +def job_to_output_dataset_association(model, session, history_dataset_association): +def job_to_output_dataset_collection_association(model, session, history_dataset_collection_association): +def job_to_output_library_dataset_association(model, session, library_dataset_dataset_association): +def library(model, session): +def library_dataset(model, session): +def library_dataset_annotation_association(model, session): +def library_dataset_collection_annotation_association(model, session): +def library_dataset_collection_association(model, session): +def library_dataset_collection_rating_association(model, session): +def library_dataset_collection_tag_association(model, session): +def library_dataset_dataset_association(model, session): +def library_dataset_dataset_association_permission(model, session, library_dataset_dataset_association, role): +def library_dataset_dataset_association_tag_association(model, session): +def library_dataset_dataset_info_association( +def library_dataset_permission(model, session, library_dataset, role): +def library_folder(model, session): +def library_folder_info_association(model, session, library_folder, form_definition, form_values): +def library_folder_permission(model, session, library_folder, role): +def library_info_association(model, session, library, form_definition, form_values): +def library_permission(model, session, library, role): +def metadata_file(model, session): +def page(model, session, user): +def page_annotation_association(model, session): +def page_rating_association(model, session): +def page_revision(model, session, page): +def page_tag_association(model, session): +def page_user_share_association(model, session): +def password_reset_token(model, session, user): +def post_job_action(model, session): +def post_job_action_association(model, session, post_job_action, job): +def quota(model, session): +def role(model, session): +def stored_workflow(model, session, user): +def stored_workflow_annotation_association(model, session): +def stored_workflow_rating_association(model, session): +def stored_workflow_tag_association(model, session): +def stored_workflow_user_share_association(model, session): +def tag(model, session): +def task(model, session, job): +def task_metric_numeric(model, session): +def task_metric_text(model, session): +def tool_tag_association(model, session): +def transfer_job(model, session): +def user(model, session): +def user_address(model, session, user): +def user_authnz_token(model, session, user): +def user_group_association(model, session): +def user_preference(model, session): +def user_quota_association(model, session): +def user_role_association(model, session, user, role): +def visualization(model, session, user): +def visualization_annotation_association(model, session): +def visualization_rating_association(model, session, user, visualization): +def visualization_revision(model, session, visualization): +def visualization_tag_association(model, session): +def visualization_user_share_association(model, session): +def workflow(model, session): +def workflow_invocation(model, session, workflow): +def workflow_invocation_output_dataset_association(model, session): +def workflow_invocation_output_dataset_collection_association(model, session): +def workflow_invocation_output_value(model, session): +def workflow_invocation_step(model, session, workflow_invocation, workflow_step): +def workflow_invocation_step_output_dataset_association(model, session): +def workflow_invocation_step_output_dataset_collection_association(model, session): +def workflow_output(model, session, workflow_step): +def workflow_request_input_parameter(model, session): +def workflow_request_input_step_parameter(model, session): +def workflow_request_step_state(model, session): +def workflow_request_to_input_dataset_association(model, session): +def workflow_request_to_input_dataset_collection_association(model, session): +def workflow_step(model, session, workflow): +def workflow_step_connection(model, session): +def workflow_step_input(model, session, workflow_step): +def workflow_step_tag_association(model, session): diff --git a/_fix-sort b/_fix-sort new file mode 100644 index 00000000000..f89eba342f7 --- /dev/null +++ b/_fix-sort @@ -0,0 +1,135 @@ +def api_keys(model, session): +def cleanup_event(model, session): +def cloud_authz(model, session, user, user_authnz_token): +def custos_authnz_token(model, session, user, user_authnz_token): +def data_manager_history_association(model, session): +def data_manager_job_association(model, session): +def dataset(model, session): +def dataset_collection(model, session): +def dataset_collection_element( +def dataset_hash(model, session): +def dataset_permission(model, session, dataset): +def dataset_source(model, session): +def dataset_source_hash(model, session): +def default_history_permissions(model, session, history, role): +def default_quota_association(model, session, quota): +def default_user_permissions(model, session, user, role): +def deferred_job(model, session): +def dynamic_tool(model, session): +def extended_metadata(model, session): +def extended_metadata_index(model, session, extended_metadata): +def form_definition(model, session, form_definition_current): +def form_definition_current(model, session): +def form_values(model, session): +def galaxy_session(model, session, user): +def galaxy_session_history_association(model, session, galaxy_session, history): +def genome_index_tool_data(model, session): +def group(model, session): +def group_quota_association(model, session): +def group_role_association(model, session): +def history(model, session): +def history_annotation_association(model, session): +def history_dataset_annotation_association(model, session): +def history_dataset_association(model, session, dataset): +def history_dataset_association_annotation_association(model, session): +def history_dataset_association_rating_association(model, session): +def history_dataset_association_tag_association(model, session): +def history_dataset_collection_annotation_association(model, session): +def history_dataset_collection_association(model, session): +def history_dataset_collection_rating_association( +def history_dataset_collection_tag_association(model, session): +def history_rating_association(model, session, user, history): +def history_tag_association(model, session): +def history_user_share_association(model, session): +def implicit_collection_jobs(model, session): +def implicit_collection_jobs_job_association(model, session): +def implicitly_converted_dataset_association(model, session, history_dataset_association): +def implicitly_created_dataset_collection_input(model, session, history_dataset_collection_association): +def interactive_tool_entry_point(model, session): +def job(model, session): +def job_container_association(model, session): +def job_export_history_archive(model, session): +def job_external_output_metadata(model, session, job, history_dataset_association): +def job_metric_numeric(model, session): +def job_metric_text(model, session): +def job_parameter(model, session): +def job_state_history(model, session, job): +def job_to_implicit_output_dataset_collection_association(model, session, dataset_collection): +def job_to_input_dataset_association(model, session, history_dataset_association): +def job_to_input_dataset_collection_association(model, session, history_dataset_collection_association): +def job_to_input_dataset_collection_element_association(model, session, dataset_collection_element): +def job_to_input_library_dataset_association(model, session, library_dataset_dataset_association): +def job_to_output_dataset_association(model, session, history_dataset_association): +def job_to_output_dataset_collection_association(model, session, history_dataset_collection_association): +def job_to_output_library_dataset_association(model, session, library_dataset_dataset_association): +def library(model, session): +def library_dataset(model, session): +def library_dataset_annotation_association(model, session): +def library_dataset_collection_annotation_association(model, session): +def library_dataset_collection_association(model, session): +def library_dataset_collection_rating_association(model, session): +def library_dataset_collection_tag_association(model, session): +def library_dataset_dataset_association(model, session): +def library_dataset_dataset_association_permission(model, session, library_dataset_dataset_association, role): +def library_dataset_dataset_association_tag_association(model, session): +def library_dataset_dataset_info_association( +def library_dataset_permission(model, session, library_dataset, role): +def library_folder(model, session): +def library_folder_info_association(model, session, library_folder, form_definition, form_values): +def library_folder_permission(model, session, library_folder, role): +def library_info_association(model, session, library, form_definition, form_values): +def library_permission(model, session, library, role): +def metadata_file(model, session): +def page(model, session, user): +def page_annotation_association(model, session): +def page_rating_association(model, session): +def page_revision(model, session, page): +def page_tag_association(model, session): +def page_user_share_association(model, session): +def password_reset_token(model, session, user): +def post_job_action(model, session): +def post_job_action_association(model, session, post_job_action, job): +def quota(model, session): +def role(model, session): +def stored_workflow(model, session, user): +def stored_workflow_annotation_association(model, session): +def stored_workflow_rating_association(model, session): +def stored_workflow_tag_association(model, session): +def stored_workflow_user_share_association(model, session): +def tag(model, session): +def task(model, session, job): +def task_metric_numeric(model, session): +def task_metric_text(model, session): +def tool_tag_association(model, session): +def transfer_job(model, session): +def user(model, session): +def user_address(model, session, user): +def user_authnz_token(model, session, user): +def user_group_association(model, session): +def user_preference(model, session): +def user_quota_association(model, session): +def user_role_association(model, session, user, role): +def visualization(model, session, user): +def visualization_annotation_association(model, session): +def visualization_rating_association(model, session, user, visualization): +def visualization_revision(model, session, visualization): +def visualization_tag_association(model, session): +def visualization_user_share_association(model, session): +def workflow(model, session): +def workflow_invocation(model, session, workflow): +def workflow_invocation_output_dataset_association(model, session): +def workflow_invocation_output_dataset_collection_association(model, session): +def workflow_invocation_output_value(model, session): +def workflow_invocation_step(model, session, workflow_invocation, workflow_step): +def workflow_invocation_step_output_dataset_association(model, session): +def workflow_invocation_step_output_dataset_collection_association(model, session): +def workflow_output(model, session, workflow_step): +def workflow_request_input_parameter(model, session): +def workflow_request_input_step_parameter(model, session): +def workflow_request_step_state(model, session): +def workflow_request_to_input_dataset_association(model, session): +def workflow_request_to_input_dataset_collection_association(model, session): +def workflow_step(model, session, workflow): +def workflow_step_connection(model, session): +def workflow_step_input(model, session, workflow_step): +def workflow_step_tag_association(model, session): diff --git a/test/unit/model/test_model.py b/test/unit/model/test_mapping.py similarity index 100% rename from test/unit/model/test_model.py rename to test/unit/model/test_mapping.py index 233fcd9b7e4..8539808c95d 100644 --- a/test/unit/model/test_model.py +++ b/test/unit/model/test_mapping.py @@ -6643,14 +6643,14 @@ def history_annotation_association(model, session): @pytest.fixture -def history_dataset_association(model, session, dataset): - instance = model.HistoryDatasetAssociation(dataset=dataset) +def history_dataset_annotation_association(model, session): + instance = model.HistoryDatasetAnnotationAssociation() yield from dbcleanup_wrapper(session, instance) @pytest.fixture -def history_dataset_annotation_association(model, session): - instance = model.HistoryDatasetAnnotationAssociation() +def history_dataset_association(model, session, dataset): + instance = model.HistoryDatasetAssociation(dataset=dataset) yield from dbcleanup_wrapper(session, instance) @@ -6673,14 +6673,14 @@ def history_dataset_association_tag_association(model, session): @pytest.fixture -def history_dataset_collection_association(model, session): - instance = model.HistoryDatasetCollectionAssociation() +def history_dataset_collection_annotation_association(model, session): + instance = model.HistoryDatasetCollectionAssociationAnnotationAssociation() yield from dbcleanup_wrapper(session, instance) @pytest.fixture -def history_dataset_collection_annotation_association(model, session): - instance = model.HistoryDatasetCollectionAssociationAnnotationAssociation() +def history_dataset_collection_association(model, session): + instance = model.HistoryDatasetCollectionAssociation() yield from dbcleanup_wrapper(session, instance) @@ -6832,14 +6832,14 @@ def job_to_input_library_dataset_association(model, session, library_dataset_dat @pytest.fixture -def job_to_output_dataset_collection_association(model, session, history_dataset_collection_association): - instance = model.JobToOutputDatasetCollectionAssociation(None, history_dataset_collection_association) +def job_to_output_dataset_association(model, session, history_dataset_association): + instance = model.JobToOutputDatasetAssociation(None, history_dataset_association) yield from dbcleanup_wrapper(session, instance) @pytest.fixture -def job_to_output_dataset_association(model, session, history_dataset_association): - instance = model.JobToOutputDatasetAssociation(None, history_dataset_association) +def job_to_output_dataset_collection_association(model, session, history_dataset_collection_association): + instance = model.JobToOutputDatasetCollectionAssociation(None, history_dataset_collection_association) yield from dbcleanup_wrapper(session, instance) @@ -6855,12 +6855,6 @@ def library(model, session): yield from dbcleanup_wrapper(session, instance) -@pytest.fixture -def library_folder(model, session): - instance = model.LibraryFolder() - yield from dbcleanup_wrapper(session, instance) - - @pytest.fixture def library_dataset(model, session): instance = model.LibraryDataset() @@ -6903,6 +6897,12 @@ def library_dataset_dataset_association(model, session): yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def library_dataset_dataset_association_permission(model, session, library_dataset_dataset_association, role): + instance = model.LibraryDatasetDatasetAssociationPermissions('a', library_dataset_dataset_association, role) + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def library_dataset_dataset_association_tag_association(model, session): instance = model.LibraryDatasetDatasetAssociationTagAssociation() @@ -6929,8 +6929,8 @@ def library_dataset_permission(model, session, library_dataset, role): @pytest.fixture -def library_dataset_dataset_association_permission(model, session, library_dataset_dataset_association, role): - instance = model.LibraryDatasetDatasetAssociationPermissions('a', library_dataset_dataset_association, role) +def library_folder(model, session): + instance = model.LibraryFolder() yield from dbcleanup_wrapper(session, instance) @@ -6971,13 +6971,6 @@ def page(model, session, user): yield from dbcleanup_wrapper(session, instance) -@pytest.fixture -def page_revision(model, session, page): - instance = model.PageRevision() - instance.page = page - yield from dbcleanup_wrapper(session, instance) - - @pytest.fixture def page_annotation_association(model, session): instance = model.PageAnnotationAssociation() @@ -6990,6 +6983,13 @@ def page_rating_association(model, session): yield from dbcleanup_wrapper(session, instance) +@pytest.fixture +def page_revision(model, session, page): + instance = model.PageRevision() + instance.page = page + yield from dbcleanup_wrapper(session, instance) + + @pytest.fixture def page_tag_association(model, session): instance = model.PageTagAssociation() @@ -7226,14 +7226,14 @@ def workflow_invocation_step(model, session, workflow_invocation, workflow_step) @pytest.fixture -def workflow_invocation_step_output_dataset_collection_association(model, session): - instance = model.WorkflowInvocationStepOutputDatasetCollectionAssociation() +def workflow_invocation_step_output_dataset_association(model, session): + instance = model.WorkflowInvocationStepOutputDatasetAssociation() yield from dbcleanup_wrapper(session, instance) @pytest.fixture -def workflow_invocation_step_output_dataset_association(model, session): - instance = model.WorkflowInvocationStepOutputDatasetAssociation() +def workflow_invocation_step_output_dataset_collection_association(model, session): + instance = model.WorkflowInvocationStepOutputDatasetCollectionAssociation() yield from dbcleanup_wrapper(session, instance) @@ -7314,16 +7314,6 @@ def dataset_collection_factory(model): return make_instance -@pytest.fixture -def history_factory(model): - def make_instance(**kwds): - instance = model.History() - if 'deleted' in kwds: - instance.deleted = kwds['deleted'] - return instance - return make_instance - - @pytest.fixture def history_dataset_association_factory(model): def make_instance(*args, **kwds): @@ -7338,6 +7328,16 @@ def history_dataset_collection_association_factory(model): return make_instance +@pytest.fixture +def history_factory(model): + def make_instance(**kwds): + instance = model.History() + if 'deleted' in kwds: + instance.deleted = kwds['deleted'] + return instance + return make_instance + + @pytest.fixture def history_rating_association_factory(model, user, history): def make_instance(*args, **kwds): From 2d58d609a66b06de40e5e9efd86bd9599524f316 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 09:42:51 -0400 Subject: [PATCH 208/225] Improve testing of uuid values --- test/unit/model/test_mapping.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 8539808c95d..6fcb07c4c86 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -819,7 +819,7 @@ class TestDynamicTool(BaseTest): tool_version = 'c' tool_path = 'd' tool_directory = 'e' - uuid = None + uuid = uuid4() active = True hidden = True value = 'f' @@ -833,7 +833,7 @@ class TestDynamicTool(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert stored_obj.uuid + assert stored_obj.uuid == uuid assert stored_obj.create_time == create_time assert stored_obj.update_time == update_time assert stored_obj.tool_id == tool_id @@ -5986,7 +5986,7 @@ class TestWorkflowOutput(BaseTest): assert cls_.__tablename__ == 'workflow_output' def test_columns(self, session, cls_, workflow_step): - output_name, label, uuid = 'a', 'b', None + output_name, label, uuid = 'a', 'b', uuid4() obj = cls_(workflow_step, output_name, label, uuid) with dbcleanup(session, obj) as obj_id: @@ -5995,7 +5995,7 @@ class TestWorkflowOutput(BaseTest): assert stored_obj.workflow_step_id == workflow_step.id assert stored_obj.output_name == output_name assert stored_obj.label == label - assert stored_obj.uuid + assert stored_obj.uuid == uuid def test_relationships(self, session, cls_, workflow_step): obj = cls_(workflow_step) @@ -6186,6 +6186,7 @@ class TestWorkflowStep(BaseTest): config = 'g' order_index = 'h' label = 'k' + uuid = uuid4() subworkflow = workflow_factory() persist(session, subworkflow) @@ -6205,6 +6206,7 @@ class TestWorkflowStep(BaseTest): obj.config = config obj.order_index = order_index obj.label = label + obj.uuid = uuid with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) @@ -6222,7 +6224,7 @@ class TestWorkflowStep(BaseTest): assert stored_obj.position == position assert stored_obj.config == config assert stored_obj.order_index == order_index - assert stored_obj.uuid # set in constructor + assert stored_obj.uuid == uuid assert stored_obj.label == label delete_from_database(session, [subworkflow]) From 4d55b6b0764607fdbb9ac10e388e46e4ad97efe2 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 11:10:47 -0400 Subject: [PATCH 209/225] Add tests for test module utility functions, modify delete Delete: accept one or multiple objects instead of list only. --- test/unit/model/test_mapping.py | 8 ++- test/unit/model/test_test_mapping.py | 79 ++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 test/unit/model/test_test_mapping.py diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 6fcb07c4c86..37425c527f2 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -7456,7 +7456,7 @@ def workflow_step_factory(model, workflow): return make_instance -# Mics. helpers. +# Test utilities def dbcleanup_wrapper(session, obj, where_clause=None): with dbcleanup(session, obj, where_clause): @@ -7502,6 +7502,12 @@ def delete_from_database(session, objects): May be called at the end of a test if use of a context manager is impractical. (Assume all objects have the id field as their primary key.) """ + # Ensure we have a list of objects + try: + objects = list(objects) + except TypeError: + objects = [objects] + for obj in objects: table = obj.__table__ stmt = delete(table).where(table.c.id == obj.id) diff --git a/test/unit/model/test_test_mapping.py b/test/unit/model/test_test_mapping.py new file mode 100644 index 00000000000..0284d312802 --- /dev/null +++ b/test/unit/model/test_test_mapping.py @@ -0,0 +1,79 @@ +""" +This module contains tests for the utility functions in the test_mapping module. +""" + +import pytest +from sqlalchemy import select +from sqlalchemy.exc import NoResultFound + +import galaxy.model.mapping as mapping +from . test_mapping import ( + delete_from_database, + persist, +) + + +@pytest.fixture(scope='module') +def model(): + db_uri = 'sqlite:///:memory:' + return mapping.init('/tmp', db_uri, create_tables=True) + + +@pytest.fixture +def session(model): + Session = model.session + yield Session() + Session.remove() # Ensures we get a new session for each test + + +def test_persist(session, model): + """ + Verify item is stored in database. + We call persist() with default arg value: `return_id=True`. Passing `False` results + in the same functionality except the return value of the funtion is `None`. + """ + cls_ = model.CleanupEvent + instance = cls_() + instance_id = persist(session, instance) # store instance in database + assert instance_id == 1 # id was assigned + assert instance not in session # instance was expunged from session + + stored_instance = _get_stored_instance_by_id(session, cls_, instance_id) + assert stored_instance.id == instance.id # instance can be retrieved by id + assert stored_instance is not instance # retrieved instance is not the same object + + +def test_delete_from_database_one_item(session, model): + """Verify item is deleted from database.""" + cls_ = model.CleanupEvent + instance = cls_() + id = persist(session, instance) # store instance in database + + stored_instance = _get_stored_instance_by_id(session, cls_, id) + assert stored_instance is not None # instance is present in database + + delete_from_database(session, instance) # delete instance from database + + with pytest.raises(NoResultFound): # instance no longer present in database + stored_instance = _get_stored_instance_by_id(session, cls_, id) + + +def test_delete_from_database_multiple_items(session, model): + """Verify multiple items are deleted from database.""" + cls_ = model.CleanupEvent + instance1 = cls_() + instance2 = cls_() + id1 = persist(session, instance1) + id2 = persist(session, instance2) + + delete_from_database(session, [instance1, instance2]) + + with pytest.raises(NoResultFound): + _get_stored_instance_by_id(session, cls_, id1) + with pytest.raises(NoResultFound): + _get_stored_instance_by_id(session, cls_, id2) + + +def _get_stored_instance_by_id(session, cls_, id): + statement = select(cls_).where(cls_.__table__.c.id == id) + return session.execute(statement).scalar_one() From dda8dcc9fc1260c895c16aeeb9d8a51ede395b94 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 13:56:41 -0400 Subject: [PATCH 210/225] Do not use Galaxy model for testing utility functions --- test/unit/model/test_mapping.py | 6 +- test/unit/model/test_test_mapping.py | 99 +++++++++++++++++++++------- 2 files changed, 77 insertions(+), 28 deletions(-) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 37425c527f2..17972300754 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -7479,7 +7479,7 @@ def dbcleanup(session, obj, where_clause=None): finally: table = obj.__table__ if where_clause is None: - where_clause = get_default_where_clause(type(obj), obj_id) + where_clause = _get_default_where_clause(type(obj), obj_id) stmt = delete(table).where(where_clause) session.execute(stmt) @@ -7518,7 +7518,7 @@ def get_stored_obj(session, cls, obj_id=None, where_clause=None, unique=False): # Either obj_id or where_clause must be provided, but not both assert bool(obj_id) ^ (where_clause is not None) if where_clause is None: - where_clause = get_default_where_clause(cls, obj_id) + where_clause = _get_default_where_clause(cls, obj_id) stmt = select(cls).where(where_clause) result = session.execute(stmt) # unique() is required if result contains joint eager loads against collections @@ -7528,7 +7528,7 @@ def get_stored_obj(session, cls, obj_id=None, where_clause=None, unique=False): return result.scalar_one() -def get_default_where_clause(cls, obj_id): +def _get_default_where_clause(cls, obj_id): where_clause = cls.__table__.c.id == obj_id return where_clause diff --git a/test/unit/model/test_test_mapping.py b/test/unit/model/test_test_mapping.py index 0284d312802..cb909878e2f 100644 --- a/test/unit/model/test_test_mapping.py +++ b/test/unit/model/test_test_mapping.py @@ -3,77 +3,126 @@ This module contains tests for the utility functions in the test_mapping module. """ import pytest -from sqlalchemy import select +from sqlalchemy import Column, create_engine, Integer, select from sqlalchemy.exc import NoResultFound +from sqlalchemy.orm import registry, Session -import galaxy.model.mapping as mapping from . test_mapping import ( delete_from_database, + get_stored_obj, persist, ) +mapper_registry = registry() + + +@mapper_registry.mapped +class Foo: + __tablename__ = 'foo' + id = Column(Integer, primary_key=True) + + @pytest.fixture(scope='module') -def model(): - db_uri = 'sqlite:///:memory:' - return mapping.init('/tmp', db_uri, create_tables=True) +def engine(): + """Create connection engine. (Fixture is module-scoped).""" + db_uri = 'sqlite:///:memory:' # We only need sqlite for these tests + return create_engine(db_uri) + + +@pytest.fixture(scope='module') +def init(engine): + """Create database objects. (Fixture is module-scoped).""" + mapper_registry.metadata.create_all(engine) @pytest.fixture -def session(model): - Session = model.session - yield Session() - Session.remove() # Ensures we get a new session for each test +def session(init, engine): + """Provides a basic session object, closed on exit.""" + with Session(engine) as s: + yield s -def test_persist(session, model): +def test_persist(session): """ Verify item is stored in database. We call persist() with default arg value: `return_id=True`. Passing `False` results in the same functionality except the return value of the funtion is `None`. """ - cls_ = model.CleanupEvent - instance = cls_() + instance = Foo() instance_id = persist(session, instance) # store instance in database assert instance_id == 1 # id was assigned assert instance not in session # instance was expunged from session - stored_instance = _get_stored_instance_by_id(session, cls_, instance_id) + stored_instance = _get_stored_instance_by_id(session, Foo, instance_id) assert stored_instance.id == instance.id # instance can be retrieved by id assert stored_instance is not instance # retrieved instance is not the same object -def test_delete_from_database_one_item(session, model): +def test_delete_from_database_one_item(session): """Verify item is deleted from database.""" - cls_ = model.CleanupEvent - instance = cls_() + instance = Foo() id = persist(session, instance) # store instance in database - stored_instance = _get_stored_instance_by_id(session, cls_, id) + stored_instance = _get_stored_instance_by_id(session, Foo, id) assert stored_instance is not None # instance is present in database delete_from_database(session, instance) # delete instance from database with pytest.raises(NoResultFound): # instance no longer present in database - stored_instance = _get_stored_instance_by_id(session, cls_, id) + stored_instance = _get_stored_instance_by_id(session, Foo, id) -def test_delete_from_database_multiple_items(session, model): +def test_delete_from_database_multiple_items(session): """Verify multiple items are deleted from database.""" - cls_ = model.CleanupEvent - instance1 = cls_() - instance2 = cls_() + instance1 = Foo() + instance2 = Foo() id1 = persist(session, instance1) id2 = persist(session, instance2) delete_from_database(session, [instance1, instance2]) with pytest.raises(NoResultFound): - _get_stored_instance_by_id(session, cls_, id1) + _get_stored_instance_by_id(session, Foo, id1) with pytest.raises(NoResultFound): - _get_stored_instance_by_id(session, cls_, id2) + _get_stored_instance_by_id(session, Foo, id2) + + +def test_get_stored_obj_must_have_obj_id_xor_where_clause(): + """ + Verify that function must be called with either obj_id or where_clause, but not both. + """ + with pytest.raises(AssertionError): + get_stored_obj(None, None, obj_id=1, where_clause='a') + with pytest.raises(AssertionError): + get_stored_obj(None, None, obj_id=None, where_clause=None) + + +def test_get_stored_obj_by_id(session): + """Verify item is retrieved from database by id.""" + instance = Foo() + id = persist(session, instance) + + stored_instance = get_stored_obj(session, Foo, id) + assert stored_instance.id == instance.id + + +def test_get_stored_obj_by_where_clause(session): + """Verify item is retrieved from database by a custom WHERE clause.""" + instance1 = Foo() + instance2 = Foo() + id1 = persist(session, instance1) + id2 = persist(session, instance2) + + where_clause = Foo.__table__.c.id == id1 + stored_instance = get_stored_obj(session, Foo, where_clause=where_clause) + assert stored_instance.id == instance1.id + + where_clause = Foo.__table__.c.id == id2 + stored_instance = get_stored_obj(session, Foo, where_clause=where_clause) + assert stored_instance.id == instance2.id def _get_stored_instance_by_id(session, cls_, id): - statement = select(cls_).where(cls_.__table__.c.id == id) + statement = select(Foo).where(cls_.__table__.c.id == id) return session.execute(statement).scalar_one() From be123b783f1e4918b0c1bf839cd848051fb395e9 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 15:09:51 -0400 Subject: [PATCH 211/225] Add remaning utility tests --- _fix | 135 ------------------ _fix-sort | 135 ------------------ test/unit/model/test_test_mapping.py | 199 +++++++++++++++++++-------- 3 files changed, 140 insertions(+), 329 deletions(-) delete mode 100644 _fix delete mode 100644 _fix-sort diff --git a/_fix b/_fix deleted file mode 100644 index f89eba342f7..00000000000 --- a/_fix +++ /dev/null @@ -1,135 +0,0 @@ -def api_keys(model, session): -def cleanup_event(model, session): -def cloud_authz(model, session, user, user_authnz_token): -def custos_authnz_token(model, session, user, user_authnz_token): -def data_manager_history_association(model, session): -def data_manager_job_association(model, session): -def dataset(model, session): -def dataset_collection(model, session): -def dataset_collection_element( -def dataset_hash(model, session): -def dataset_permission(model, session, dataset): -def dataset_source(model, session): -def dataset_source_hash(model, session): -def default_history_permissions(model, session, history, role): -def default_quota_association(model, session, quota): -def default_user_permissions(model, session, user, role): -def deferred_job(model, session): -def dynamic_tool(model, session): -def extended_metadata(model, session): -def extended_metadata_index(model, session, extended_metadata): -def form_definition(model, session, form_definition_current): -def form_definition_current(model, session): -def form_values(model, session): -def galaxy_session(model, session, user): -def galaxy_session_history_association(model, session, galaxy_session, history): -def genome_index_tool_data(model, session): -def group(model, session): -def group_quota_association(model, session): -def group_role_association(model, session): -def history(model, session): -def history_annotation_association(model, session): -def history_dataset_annotation_association(model, session): -def history_dataset_association(model, session, dataset): -def history_dataset_association_annotation_association(model, session): -def history_dataset_association_rating_association(model, session): -def history_dataset_association_tag_association(model, session): -def history_dataset_collection_annotation_association(model, session): -def history_dataset_collection_association(model, session): -def history_dataset_collection_rating_association( -def history_dataset_collection_tag_association(model, session): -def history_rating_association(model, session, user, history): -def history_tag_association(model, session): -def history_user_share_association(model, session): -def implicit_collection_jobs(model, session): -def implicit_collection_jobs_job_association(model, session): -def implicitly_converted_dataset_association(model, session, history_dataset_association): -def implicitly_created_dataset_collection_input(model, session, history_dataset_collection_association): -def interactive_tool_entry_point(model, session): -def job(model, session): -def job_container_association(model, session): -def job_export_history_archive(model, session): -def job_external_output_metadata(model, session, job, history_dataset_association): -def job_metric_numeric(model, session): -def job_metric_text(model, session): -def job_parameter(model, session): -def job_state_history(model, session, job): -def job_to_implicit_output_dataset_collection_association(model, session, dataset_collection): -def job_to_input_dataset_association(model, session, history_dataset_association): -def job_to_input_dataset_collection_association(model, session, history_dataset_collection_association): -def job_to_input_dataset_collection_element_association(model, session, dataset_collection_element): -def job_to_input_library_dataset_association(model, session, library_dataset_dataset_association): -def job_to_output_dataset_association(model, session, history_dataset_association): -def job_to_output_dataset_collection_association(model, session, history_dataset_collection_association): -def job_to_output_library_dataset_association(model, session, library_dataset_dataset_association): -def library(model, session): -def library_dataset(model, session): -def library_dataset_annotation_association(model, session): -def library_dataset_collection_annotation_association(model, session): -def library_dataset_collection_association(model, session): -def library_dataset_collection_rating_association(model, session): -def library_dataset_collection_tag_association(model, session): -def library_dataset_dataset_association(model, session): -def library_dataset_dataset_association_permission(model, session, library_dataset_dataset_association, role): -def library_dataset_dataset_association_tag_association(model, session): -def library_dataset_dataset_info_association( -def library_dataset_permission(model, session, library_dataset, role): -def library_folder(model, session): -def library_folder_info_association(model, session, library_folder, form_definition, form_values): -def library_folder_permission(model, session, library_folder, role): -def library_info_association(model, session, library, form_definition, form_values): -def library_permission(model, session, library, role): -def metadata_file(model, session): -def page(model, session, user): -def page_annotation_association(model, session): -def page_rating_association(model, session): -def page_revision(model, session, page): -def page_tag_association(model, session): -def page_user_share_association(model, session): -def password_reset_token(model, session, user): -def post_job_action(model, session): -def post_job_action_association(model, session, post_job_action, job): -def quota(model, session): -def role(model, session): -def stored_workflow(model, session, user): -def stored_workflow_annotation_association(model, session): -def stored_workflow_rating_association(model, session): -def stored_workflow_tag_association(model, session): -def stored_workflow_user_share_association(model, session): -def tag(model, session): -def task(model, session, job): -def task_metric_numeric(model, session): -def task_metric_text(model, session): -def tool_tag_association(model, session): -def transfer_job(model, session): -def user(model, session): -def user_address(model, session, user): -def user_authnz_token(model, session, user): -def user_group_association(model, session): -def user_preference(model, session): -def user_quota_association(model, session): -def user_role_association(model, session, user, role): -def visualization(model, session, user): -def visualization_annotation_association(model, session): -def visualization_rating_association(model, session, user, visualization): -def visualization_revision(model, session, visualization): -def visualization_tag_association(model, session): -def visualization_user_share_association(model, session): -def workflow(model, session): -def workflow_invocation(model, session, workflow): -def workflow_invocation_output_dataset_association(model, session): -def workflow_invocation_output_dataset_collection_association(model, session): -def workflow_invocation_output_value(model, session): -def workflow_invocation_step(model, session, workflow_invocation, workflow_step): -def workflow_invocation_step_output_dataset_association(model, session): -def workflow_invocation_step_output_dataset_collection_association(model, session): -def workflow_output(model, session, workflow_step): -def workflow_request_input_parameter(model, session): -def workflow_request_input_step_parameter(model, session): -def workflow_request_step_state(model, session): -def workflow_request_to_input_dataset_association(model, session): -def workflow_request_to_input_dataset_collection_association(model, session): -def workflow_step(model, session, workflow): -def workflow_step_connection(model, session): -def workflow_step_input(model, session, workflow_step): -def workflow_step_tag_association(model, session): diff --git a/_fix-sort b/_fix-sort deleted file mode 100644 index f89eba342f7..00000000000 --- a/_fix-sort +++ /dev/null @@ -1,135 +0,0 @@ -def api_keys(model, session): -def cleanup_event(model, session): -def cloud_authz(model, session, user, user_authnz_token): -def custos_authnz_token(model, session, user, user_authnz_token): -def data_manager_history_association(model, session): -def data_manager_job_association(model, session): -def dataset(model, session): -def dataset_collection(model, session): -def dataset_collection_element( -def dataset_hash(model, session): -def dataset_permission(model, session, dataset): -def dataset_source(model, session): -def dataset_source_hash(model, session): -def default_history_permissions(model, session, history, role): -def default_quota_association(model, session, quota): -def default_user_permissions(model, session, user, role): -def deferred_job(model, session): -def dynamic_tool(model, session): -def extended_metadata(model, session): -def extended_metadata_index(model, session, extended_metadata): -def form_definition(model, session, form_definition_current): -def form_definition_current(model, session): -def form_values(model, session): -def galaxy_session(model, session, user): -def galaxy_session_history_association(model, session, galaxy_session, history): -def genome_index_tool_data(model, session): -def group(model, session): -def group_quota_association(model, session): -def group_role_association(model, session): -def history(model, session): -def history_annotation_association(model, session): -def history_dataset_annotation_association(model, session): -def history_dataset_association(model, session, dataset): -def history_dataset_association_annotation_association(model, session): -def history_dataset_association_rating_association(model, session): -def history_dataset_association_tag_association(model, session): -def history_dataset_collection_annotation_association(model, session): -def history_dataset_collection_association(model, session): -def history_dataset_collection_rating_association( -def history_dataset_collection_tag_association(model, session): -def history_rating_association(model, session, user, history): -def history_tag_association(model, session): -def history_user_share_association(model, session): -def implicit_collection_jobs(model, session): -def implicit_collection_jobs_job_association(model, session): -def implicitly_converted_dataset_association(model, session, history_dataset_association): -def implicitly_created_dataset_collection_input(model, session, history_dataset_collection_association): -def interactive_tool_entry_point(model, session): -def job(model, session): -def job_container_association(model, session): -def job_export_history_archive(model, session): -def job_external_output_metadata(model, session, job, history_dataset_association): -def job_metric_numeric(model, session): -def job_metric_text(model, session): -def job_parameter(model, session): -def job_state_history(model, session, job): -def job_to_implicit_output_dataset_collection_association(model, session, dataset_collection): -def job_to_input_dataset_association(model, session, history_dataset_association): -def job_to_input_dataset_collection_association(model, session, history_dataset_collection_association): -def job_to_input_dataset_collection_element_association(model, session, dataset_collection_element): -def job_to_input_library_dataset_association(model, session, library_dataset_dataset_association): -def job_to_output_dataset_association(model, session, history_dataset_association): -def job_to_output_dataset_collection_association(model, session, history_dataset_collection_association): -def job_to_output_library_dataset_association(model, session, library_dataset_dataset_association): -def library(model, session): -def library_dataset(model, session): -def library_dataset_annotation_association(model, session): -def library_dataset_collection_annotation_association(model, session): -def library_dataset_collection_association(model, session): -def library_dataset_collection_rating_association(model, session): -def library_dataset_collection_tag_association(model, session): -def library_dataset_dataset_association(model, session): -def library_dataset_dataset_association_permission(model, session, library_dataset_dataset_association, role): -def library_dataset_dataset_association_tag_association(model, session): -def library_dataset_dataset_info_association( -def library_dataset_permission(model, session, library_dataset, role): -def library_folder(model, session): -def library_folder_info_association(model, session, library_folder, form_definition, form_values): -def library_folder_permission(model, session, library_folder, role): -def library_info_association(model, session, library, form_definition, form_values): -def library_permission(model, session, library, role): -def metadata_file(model, session): -def page(model, session, user): -def page_annotation_association(model, session): -def page_rating_association(model, session): -def page_revision(model, session, page): -def page_tag_association(model, session): -def page_user_share_association(model, session): -def password_reset_token(model, session, user): -def post_job_action(model, session): -def post_job_action_association(model, session, post_job_action, job): -def quota(model, session): -def role(model, session): -def stored_workflow(model, session, user): -def stored_workflow_annotation_association(model, session): -def stored_workflow_rating_association(model, session): -def stored_workflow_tag_association(model, session): -def stored_workflow_user_share_association(model, session): -def tag(model, session): -def task(model, session, job): -def task_metric_numeric(model, session): -def task_metric_text(model, session): -def tool_tag_association(model, session): -def transfer_job(model, session): -def user(model, session): -def user_address(model, session, user): -def user_authnz_token(model, session, user): -def user_group_association(model, session): -def user_preference(model, session): -def user_quota_association(model, session): -def user_role_association(model, session, user, role): -def visualization(model, session, user): -def visualization_annotation_association(model, session): -def visualization_rating_association(model, session, user, visualization): -def visualization_revision(model, session, visualization): -def visualization_tag_association(model, session): -def visualization_user_share_association(model, session): -def workflow(model, session): -def workflow_invocation(model, session, workflow): -def workflow_invocation_output_dataset_association(model, session): -def workflow_invocation_output_dataset_collection_association(model, session): -def workflow_invocation_output_value(model, session): -def workflow_invocation_step(model, session, workflow_invocation, workflow_step): -def workflow_invocation_step_output_dataset_association(model, session): -def workflow_invocation_step_output_dataset_collection_association(model, session): -def workflow_output(model, session, workflow_step): -def workflow_request_input_parameter(model, session): -def workflow_request_input_step_parameter(model, session): -def workflow_request_step_state(model, session): -def workflow_request_to_input_dataset_association(model, session): -def workflow_request_to_input_dataset_collection_association(model, session): -def workflow_step(model, session, workflow): -def workflow_step_connection(model, session): -def workflow_step_input(model, session, workflow_step): -def workflow_step_tag_association(model, session): diff --git a/test/unit/model/test_test_mapping.py b/test/unit/model/test_test_mapping.py index cb909878e2f..eca626f6b4e 100644 --- a/test/unit/model/test_test_mapping.py +++ b/test/unit/model/test_test_mapping.py @@ -2,47 +2,31 @@ This module contains tests for the utility functions in the test_mapping module. """ +from contextlib import contextmanager + import pytest -from sqlalchemy import Column, create_engine, Integer, select +from sqlalchemy import ( + Column, + create_engine, + Index, + Integer, + select, + UniqueConstraint +) from sqlalchemy.exc import NoResultFound from sqlalchemy.orm import registry, Session from . test_mapping import ( + dbcleanup, + dbcleanup_wrapper, delete_from_database, get_stored_obj, + has_index, + has_unique_constraint, persist, ) -mapper_registry = registry() - - -@mapper_registry.mapped -class Foo: - __tablename__ = 'foo' - id = Column(Integer, primary_key=True) - - -@pytest.fixture(scope='module') -def engine(): - """Create connection engine. (Fixture is module-scoped).""" - db_uri = 'sqlite:///:memory:' # We only need sqlite for these tests - return create_engine(db_uri) - - -@pytest.fixture(scope='module') -def init(engine): - """Create database objects. (Fixture is module-scoped).""" - mapper_registry.metadata.create_all(engine) - - -@pytest.fixture -def session(init, engine): - """Provides a basic session object, closed on exit.""" - with Session(engine) as s: - yield s - - def test_persist(session): """ Verify item is stored in database. @@ -59,35 +43,6 @@ def test_persist(session): assert stored_instance is not instance # retrieved instance is not the same object -def test_delete_from_database_one_item(session): - """Verify item is deleted from database.""" - instance = Foo() - id = persist(session, instance) # store instance in database - - stored_instance = _get_stored_instance_by_id(session, Foo, id) - assert stored_instance is not None # instance is present in database - - delete_from_database(session, instance) # delete instance from database - - with pytest.raises(NoResultFound): # instance no longer present in database - stored_instance = _get_stored_instance_by_id(session, Foo, id) - - -def test_delete_from_database_multiple_items(session): - """Verify multiple items are deleted from database.""" - instance1 = Foo() - instance2 = Foo() - id1 = persist(session, instance1) - id2 = persist(session, instance2) - - delete_from_database(session, [instance1, instance2]) - - with pytest.raises(NoResultFound): - _get_stored_instance_by_id(session, Foo, id1) - with pytest.raises(NoResultFound): - _get_stored_instance_by_id(session, Foo, id2) - - def test_get_stored_obj_must_have_obj_id_xor_where_clause(): """ Verify that function must be called with either obj_id or where_clause, but not both. @@ -123,6 +78,132 @@ def test_get_stored_obj_by_where_clause(session): assert stored_instance.id == instance2.id +def test_delete_from_database_one_item(session): + """Verify item is deleted from database.""" + instance = Foo() + id = persist(session, instance) # store instance in database + + stored_instance = _get_stored_instance_by_id(session, Foo, id) + assert stored_instance is not None # instance is present in database + + delete_from_database(session, instance) # delete instance from database + + with pytest.raises(NoResultFound): # instance no longer present in database + stored_instance = _get_stored_instance_by_id(session, Foo, id) + + +def test_delete_from_database_multiple_items(session): + """Verify multiple items are deleted from database.""" + instance1 = Foo() + instance2 = Foo() + id1 = persist(session, instance1) + id2 = persist(session, instance2) + + delete_from_database(session, [instance1, instance2]) + + with pytest.raises(NoResultFound): + _get_stored_instance_by_id(session, Foo, id1) + with pytest.raises(NoResultFound): + _get_stored_instance_by_id(session, Foo, id2) + + +def test_dbcleanup_by_id(session): + instance = Foo() + with dbcleanup(session, instance) as instance_id: + stored_instance = _get_stored_instance_by_id(session, Foo, instance_id) + assert stored_instance # has been stored in the database + + with pytest.raises(NoResultFound): # has been deleted from the database + _get_stored_instance_by_id(session, Foo, instance_id) + + +def test_dbcleanup_by_where_clause(session): + instance = Foo() + where_clause = Foo.__table__.c.id == 1 + + with dbcleanup(session, instance, where_clause): + stored_instance = _get_stored_instance_by_id(session, Foo, where_clause) + assert stored_instance # has been stored in the database + + with pytest.raises(NoResultFound): # has been deleted from the database + _get_stored_instance_by_id(session, Foo, stored_instance.id) + + +def test_dbcleanup_wrapper(session): + """ + Verify dbcleanup_wrapper has same effect as dbcleanup context manager, + and yields the object instance. + """ + @contextmanager # we need a context manager to similate scope + def managed(a, b): + yield from dbcleanup_wrapper(a, b) + + instance = Foo() + # this will call dbcleanup_wrapper that stores instance in the database and returns a reference to it + with managed(session, instance) as instance2: + assert instance2 is instance # instance2 should be a reference to instance + stored_instance = _get_stored_instance_by_id(session, Foo, instance.id) + assert stored_instance # has been stored in the database + + # on exit we expect the entity to be deleted from the database + with pytest.raises(NoResultFound): # has been deleted from the database + _get_stored_instance_by_id(session, Foo, instance.id) + + +def test_has_index(session): + assert has_index(Bar.__table__, ('field1',)) + assert not has_index(Foo.__table__, ('field1',)) + + +def test_has_unique_constraint(session): + assert has_unique_constraint(Bar.__table__, ('field2',)) + assert not has_unique_constraint(Foo.__table__, ('field1',)) + + +# Test utilities + +mapper_registry = registry() + + +@mapper_registry.mapped +class Foo: + __tablename__ = 'foo' + id = Column(Integer, primary_key=True) + field1 = Column(Integer) + + +@mapper_registry.mapped +class Bar: + __tablename__ = 'bar' + id = Column(Integer, primary_key=True) + field1 = Column(Integer) + field2 = Column(Integer) + __table_args__ = ( + Index('ix', 'field1'), + UniqueConstraint('field2'), + ) + + +@pytest.fixture(scope='module') +def engine(): + """Create connection engine. (Fixture is module-scoped).""" + db_uri = 'sqlite:///:memory:' # We only need sqlite for these tests + return create_engine(db_uri) + + +@pytest.fixture(scope='module') +def init(engine): + """Create database objects. (Fixture is module-scoped).""" + mapper_registry.metadata.create_all(engine) + + +@pytest.fixture +def session(init, engine): + """Provides a basic session object, closed on exit.""" + with Session(engine) as s: + yield s + + def _get_stored_instance_by_id(session, cls_, id): statement = select(Foo).where(cls_.__table__.c.id == id) return session.execute(statement).scalar_one() From ea01d3fd23ed1299cccb7101861e3d7ddfc1777c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 17:43:27 -0400 Subject: [PATCH 212/225] Ensure all persisted objects are deleted on exit --- test/unit/model/test_mapping.py | 63 ++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 17972300754..6b6d55a50fb 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -450,7 +450,7 @@ class TestDataset(BaseTest): hda.purged = True obj.history_associations.append(hda) - with dbcleanup(session, obj) as obj_id, dbcleanup(session, hda): + with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.job.id == job.id assert stored_obj.actions == [dataset_permission] @@ -463,6 +463,8 @@ class TestDataset(BaseTest): assert len(stored_obj.history_associations) == 2 # TODO: test membership in history_associations by id + delete_from_database(session, hda) + class TestDatasetCollection(BaseTest): @@ -549,7 +551,7 @@ class TestDatasetCollectionElement(BaseTest): assert stored_obj.element_index == element_index assert stored_obj.element_identifier == element_identifier - delete_from_database(session, [parent_collection]) # because this won't be removed automatically + delete_from_database(session, parent_collection) def test_relationships( self, @@ -575,7 +577,7 @@ class TestDatasetCollectionElement(BaseTest): assert stored_obj.child_collection.id == dataset_collection.id assert stored_obj.collection.id == parent_collection.id - delete_from_database(session, [parent_collection]) # because this won't be removed automatically + delete_from_database(session, parent_collection) class TestDatasetHash(BaseTest): @@ -1563,6 +1565,7 @@ class TestHistoryDatasetAssociation(BaseTest): copied_to_ldda = library_dataset_dataset_association_factory() icda = implicitly_converted_dataset_association_factory() icpda = implicitly_converted_dataset_association_factory() + persisted = [copied_from_hda, copied_to_hda, copied_from_ldda, copied_to_ldda, icda, icpda] obj = cls_() obj.history = history @@ -1601,7 +1604,7 @@ class TestHistoryDatasetAssociation(BaseTest): assert stored_obj.implicitly_converted_datasets == [icda] assert stored_obj.implicitly_converted_parent_datasets == [icpda] - delete_from_database(session, [copied_from_hda, copied_from_ldda, copied_to_ldda, icda, icpda]) + delete_from_database(session, persisted) class TestHistoryDatasetAssociationAnnotationAssociation(BaseTest): @@ -1744,7 +1747,7 @@ class TestHistoryDatasetAssociationSubset(BaseTest): assert stored_obj.history_dataset_association_subset_id == hda_subset.id assert stored_obj.location == location - delete_from_database(session, [hda_subset]) + delete_from_database(session, hda_subset) def test_relationships( self, @@ -1763,7 +1766,7 @@ class TestHistoryDatasetAssociationSubset(BaseTest): assert stored_obj.hda.id == history_dataset_association.id assert stored_obj.subset.id == hda_subset.id - delete_from_database(session, [hda_subset]) + delete_from_database(session, hda_subset) class TestHistoryDatasetAssociationTagAssociation(BaseTest): @@ -1901,6 +1904,8 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): assert stored_obj.job_state_summary # this is a view; TODO: can we test this better? assert stored_obj.hidden_dataset_instances == [history_dataset_association] + delete_from_database(session, copied_to_hdca) + class TestHistoryDatasetCollectionAssociationAnnotationAssociation(BaseTest): @@ -3094,14 +3099,17 @@ class TestLibraryDataset(BaseTest): ldda = library_dataset_dataset_association_factory() ldda.library_dataset = obj obj.actions.append(library_dataset_permission) + persist(session, ldda) - with dbcleanup(session, obj) as obj_id, dbcleanup(session, ldda): + with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id assert stored_obj.folder.id == library_folder.id assert stored_obj.expired_datasets[0].id == ldda.id assert stored_obj.actions == [library_dataset_permission] + delete_from_database(session, ldda) + class TestLibraryDatasetCollectionAnnotationAssociation(BaseTest): @@ -3598,10 +3606,10 @@ class TestLibraryFolder(BaseTest): obj.info_association.append(library_folder_info_association) # There's no back reference, so dataset does not update folder; so we have to flush to the database - # via dbcleanup() context manager. library_dataset.folder = obj + persist(session, library_dataset) - with dbcleanup(session, obj) as obj_id, dbcleanup(session, library_dataset): + with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.parent.id == library_folder.id assert stored_obj.folders == [folder1] @@ -3614,6 +3622,8 @@ class TestLibraryFolder(BaseTest): assert stored_obj.dataset_collections == [library_dataset_collection_association] assert stored_obj.info_association == [library_folder_info_association] + delete_from_database(session, folder1) + class TestLibraryFolderInfoAssociation(BaseTest): @@ -4365,9 +4375,15 @@ class TestStoredWorkflow(BaseTest): assert stored_obj.owner_tags == [tag_assoc2] assert stored_obj.users_shared_with == [stored_workflow_user_share_association] - delete_from_database(session, [wf]) + delete_from_database(session, [wf, tag_assoc2]) - def test_average_rating(self, session, stored_workflow, user, stored_workflow_rating_association_factory): + def test_average_rating( + self, + session, + stored_workflow, + user, + stored_workflow_rating_association_factory + ): # StoredWorkflow has been expunged; to access its deferred properties, # it needs to be added back to the session. session.add(stored_workflow) @@ -5234,6 +5250,8 @@ class TestVisualization(BaseTest): assert stored_obj.average_rating == visualization_rating_association.rating assert stored_obj.users_shared_with == [visualization_user_share_association] + delete_from_database(session, revision2) + def test_average_rating(self, session, visualization, user, visualization_rating_association_factory): # Visualization has been expunged; to access its deferred properties, # it needs to be added back to the session. @@ -5569,6 +5587,8 @@ class TestWorkflowInvocation(BaseTest): assert stored_obj.parent_workflow_invocation_association == [parent_workflow_invocation_assoc] assert stored_obj.output_values == [workflow_invocation_output_value] + delete_from_database(session, [subworkflow_invocation_assoc, parent_workflow_invocation_assoc]) + class TestWorkflowInvocationOutputDatasetAssociation(BaseTest): @@ -5954,7 +5974,7 @@ class TestWorkflowInvocationToSubworkflowInvocationAssociation(BaseTest): assert stored_obj.subworkflow_invocation_id == subworkflow_invocation.id assert stored_obj.workflow_step_id == workflow_step.id - delete_from_database(session, [subworkflow_invocation]) + delete_from_database(session, subworkflow_invocation) def test_relationships( self, @@ -5979,6 +5999,8 @@ class TestWorkflowInvocationToSubworkflowInvocationAssociation(BaseTest): assert stored_obj.workflow_step.id == workflow_step.id assert stored_obj.parent_workflow_invocation.id == parent_workflow_invocation.id + delete_from_database(session, parent_workflow_invocation) + class TestWorkflowOutput(BaseTest): @@ -6227,7 +6249,7 @@ class TestWorkflowStep(BaseTest): assert stored_obj.uuid == uuid assert stored_obj.label == label - delete_from_database(session, [subworkflow]) + delete_from_database(session, subworkflow) def test_relationships( self, @@ -6241,9 +6263,9 @@ class TestWorkflowStep(BaseTest): subworkflow = workflow_factory() workflow_step_connection_in = workflow_step_connection_factory() workflow_step_connection_out = workflow_step_connection_factory() - extra_objects = [subworkflow, workflow_step_connection_in, workflow_step_connection_out] - for eo in extra_objects: - persist(session, eo) + persist(session, subworkflow) + persist(session, workflow_step_connection_in) + persist(session, workflow_step_connection_out) obj = cls_() obj.workflow = workflow @@ -6260,7 +6282,8 @@ class TestWorkflowStep(BaseTest): assert stored_obj.parent_workflow_input_connections == [workflow_step_connection_in] assert stored_obj.output_connections == [workflow_step_connection_out] - delete_from_database(session, extra_objects) + persisted = [subworkflow, workflow_step_connection_in, workflow_step_connection_out] + delete_from_database(session, persisted) class TestWorkflowStepAnnotationAssociation(BaseTest): @@ -7502,10 +7525,8 @@ def delete_from_database(session, objects): May be called at the end of a test if use of a context manager is impractical. (Assume all objects have the id field as their primary key.) """ - # Ensure we have a list of objects - try: - objects = list(objects) - except TypeError: + # Ensure we have a list of objects (check for list explicitly: a model can be iterable) + if not isinstance(objects, list): objects = [objects] for obj in objects: From b2dbac31c186fcb94d7adcae45aa17bda03dc905 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 17:53:30 -0400 Subject: [PATCH 213/225] Use dummy enum values for test isntances Because we are not testing the model logic; so we don't want to depend on stuff we don't have to. --- test/unit/model/test_mapping.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 6b6d55a50fb..a08fa4eb699 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -784,10 +784,10 @@ class TestDeferredJob(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'deferred_job' - def test_columns(self, session, cls_, model): + def test_columns(self, session, cls_): create_time = datetime.now() update_time = create_time + timedelta(hours=1) - state, plugin, params = model.DeferredJob.states.NEW, 'a', 'b' + state, plugin, params = 'a', 'b', 'c' obj = cls_(state, plugin, params) obj.create_time = create_time obj.update_time = update_time @@ -2082,8 +2082,8 @@ class TestImplicitCollectionJobs(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'implicit_collection_jobs' - def test_columns(self, session, cls_, model): - populated_state = model.ImplicitCollectionJobs.populated_states.NEW + def test_columns(self, session, cls_): + populated_state = 'a' obj = cls_() obj.populated_state = populated_state @@ -4620,11 +4620,11 @@ class TestTask(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'task' - def test_columns(self, session, cls_, model, job): + def test_columns(self, session, cls_, job): create_time = datetime.now() execution_time = create_time + timedelta(hours=1) update_time = execution_time + timedelta(hours=1) - state = model.Task.states.WAITING + state = 'p' command_line = 'a' param_filename = 'b' runner_name = 'c' @@ -4780,10 +4780,10 @@ class TestTransferJob(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'transfer_job' - def test_columns(self, session, cls_, model): + def test_columns(self, session, cls_): create_time = datetime.now() update_time = create_time + timedelta(hours=1) - state, path, info, pid, socket, params = model.TransferJob.states.NEW, 'a', 'b', 2, 3, 'c' + state, path, info, pid, socket, params = 'd', 'a', 'b', 2, 3, 'c' obj = cls_(state, path, info, pid, socket, params) obj.create_time = create_time obj.update_time = update_time From 164404188726c4a261c5191d1cf28c0f1e9befd3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 18:03:06 -0400 Subject: [PATCH 214/225] Factor out average rating testing --- test/unit/model/test_mapping.py | 72 +++++++++------------------------ 1 file changed, 20 insertions(+), 52 deletions(-) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index a08fa4eb699..dc613614c99 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -1379,19 +1379,7 @@ class TestHistory(BaseTest): assert stored_obj.jobs == [job] def test_average_rating(self, session, history, user, history_rating_association_factory): - # History has been expunged; to access its deferred properties, - # it needs to be added back to the session. - session.add(history) - assert history.average_rating is None # With no ratings, we expect None. - # Create ratings - to_cleanup = [] - for rating in (1, 2, 3, 4, 5): - history_rating_assoc = history_rating_association_factory(user, history, rating) - persist(session, history_rating_assoc) - to_cleanup.append(history_rating_assoc) - assert history.average_rating == 3.0 # Expect average after ratings added. - # Cleanup: remove ratings from database - delete_from_database(session, to_cleanup) + _run_average_rating_test(session, history, user, history_rating_association_factory) class TestHistoryAnnotationAssociation(BaseTest): @@ -3947,19 +3935,7 @@ class TestPage(BaseTest): assert stored_obj.average_rating == page_rating_association.rating def test_average_rating(self, session, page, user, page_rating_association_factory): - # Page has been expunged; to access its deferred properties, - # it needs to be added back to the session. - session.add(page) - assert page.average_rating is None # With no ratings, we expect None. - # Create ratings - to_cleanup = [] - for rating in (1, 2, 3, 4, 5): - page_rating_assoc = page_rating_association_factory(user, page, rating) - persist(session, page_rating_assoc) - to_cleanup.append(page_rating_assoc) - assert page.average_rating == 3.0 # Expect average after ratings added. - # Cleanup: remove ratings from database - delete_from_database(session, to_cleanup) + _run_average_rating_test(session, page, user, page_rating_association_factory) class TestPageAnnotationAssociation(BaseTest): @@ -4384,19 +4360,7 @@ class TestStoredWorkflow(BaseTest): user, stored_workflow_rating_association_factory ): - # StoredWorkflow has been expunged; to access its deferred properties, - # it needs to be added back to the session. - session.add(stored_workflow) - assert stored_workflow.average_rating is None # With no ratings, we expect None. - # Create ratings - to_cleanup = [] - for rating in (1, 2, 3, 4, 5): - sw_rating_assoc = stored_workflow_rating_association_factory(user, stored_workflow, rating) - persist(session, sw_rating_assoc) - to_cleanup.append(sw_rating_assoc) - assert stored_workflow.average_rating == 3.0 # Expect average after ratings added. - # Cleanup: remove ratings from database - delete_from_database(session, to_cleanup) + _run_average_rating_test(session, stored_workflow, user, stored_workflow_rating_association_factory) class TestStoredWorkflowAnnotationAssociation(BaseTest): @@ -5253,19 +5217,7 @@ class TestVisualization(BaseTest): delete_from_database(session, revision2) def test_average_rating(self, session, visualization, user, visualization_rating_association_factory): - # Visualization has been expunged; to access its deferred properties, - # it needs to be added back to the session. - session.add(visualization) - assert visualization.average_rating is None # With no ratings, we expect None. - # Create ratings - to_cleanup = [] - for rating in (1, 2, 3, 4, 5): - visualization_rating_assoc = visualization_rating_association_factory(user, visualization, rating) - persist(session, visualization_rating_assoc) - to_cleanup.append(visualization_rating_assoc) - assert visualization.average_rating == 3.0 # Expect average after ratings added. - # Cleanup: remove ratings from database - delete_from_database(session, to_cleanup) + _run_average_rating_test(session, visualization, user, visualization_rating_association_factory) class TestVisualizationAnnotationAssociation(BaseTest): @@ -7572,3 +7524,19 @@ def has_index(table, fields): def get_unique_value(): """Generate unique values to accommodate unique constraints.""" return uuid4().hex + + +def _run_average_rating_test(session, obj, user, obj_rating_association_factory): + # obj has been expunged; to access its deferred properties, + # it needs to be added back to the session. + session.add(obj) + assert obj.average_rating is None # With no ratings, we expect None. + # Create ratings + to_cleanup = [] + for rating in (1, 2, 3, 4, 5): + obj_rating_assoc = obj_rating_association_factory(user, obj, rating) + persist(session, obj_rating_assoc) + to_cleanup.append(obj_rating_assoc) + assert obj.average_rating == 3.0 # Expect average after ratings added. + # Cleanup: remove ratings from database + delete_from_database(session, to_cleanup) From f3f8dff141d07fff9d15125f79e60ad08a9c7569 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 18:09:17 -0400 Subject: [PATCH 215/225] Cleanup: update lazy load comments for consistency --- test/unit/model/test_mapping.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index dc613614c99..f6a183436bd 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -353,7 +353,7 @@ class TestDataManagerJobAssociation(BaseTest): obj = cls_() obj.create_time = create_time obj.update_time = update_time - session.add(job) + session.add(job) # must be bound to a session for lazy load of attributes obj.job = job obj.data_manager_id = data_manager_id @@ -366,7 +366,7 @@ class TestDataManagerJobAssociation(BaseTest): assert stored_obj.data_manager_id == data_manager_id def test_relationships(self, session, cls_, job): - session.add(job) + session.add(job) # must be bound to a session for lazy load of attributes obj = cls_(job=job) with dbcleanup(session, obj) as obj_id: @@ -2109,7 +2109,7 @@ class TestImplicitCollectionJobsJobAssociation(BaseTest): order_index = 1 obj = cls_() obj.implicit_collection_jobs = implicit_collection_jobs - session.add(job) + session.add(job) # must be bound to a session for lazy load of attributes obj.job = job obj.order_index = order_index @@ -2123,7 +2123,7 @@ class TestImplicitCollectionJobsJobAssociation(BaseTest): def test_relationships(self, session, cls_, implicit_collection_jobs, job): obj = cls_() obj.implicit_collection_jobs = implicit_collection_jobs - session.add(job) + session.add(job) # must be bound to a session for lazy load of attributes obj.job = job obj.order_index = 1 @@ -2518,7 +2518,7 @@ class TestJobContainerAssociation(BaseTest): created_time = datetime.now() modified_time = created_time + timedelta(hours=1) - session.add(job) + session.add(job) # must be bound to a session for lazy load of attributes obj = cls_(job, container_type, container_name, container_info) obj.created_time = created_time obj.modified_time = modified_time @@ -2534,7 +2534,7 @@ class TestJobContainerAssociation(BaseTest): assert stored_obj.modified_time == modified_time def test_relationships(self, session, cls_, job): - session.add(job) + session.add(job) # must be bound to a session for lazy load of attributes obj = cls_(job, None, None, None) with dbcleanup(session, obj) as obj_id: @@ -3024,7 +3024,7 @@ class TestLibrary(BaseTest): ): obj = cls_(None, None, None, library_folder) obj.actions.append(library_permission) - session.add(library_info_association) # must be bound to a session for lazy load of attr. `library` (https://docs.sqlalchemy.org/en/14/errors.html#error-bhk3) + session.add(library_info_association) # must be bound to a session for lazy load of attributes obj.info_association.append(library_info_association) with dbcleanup(session, obj) as obj_id: @@ -5719,8 +5719,8 @@ class TestWorkflowInvocationStep(BaseTest): update_time = create_time + timedelta(hours=1) state, action = 'a', 'b' - session.add(job) # needed for a lazy load of an attr - session.add(implicit_collection_jobs) # needed for a lazy load of an attr + session.add(job) # must be bound to a session for lazy load of attributes + session.add(implicit_collection_jobs) # must be bound to a session for lazy load of attributes obj = cls_() obj.create_time = create_time @@ -5756,8 +5756,8 @@ class TestWorkflowInvocationStep(BaseTest): workflow_invocation_step_output_dataset_association, workflow_invocation_output_value, ): - session.add(job) # needed for a lazy load of an attr - session.add(implicit_collection_jobs) # needed for a lazy load of an attr + session.add(job) # must be bound to a session for lazy load of attributes + session.add(implicit_collection_jobs) # must be bound to a session for lazy load of attributes # setup workflow_invocation_output_value to test the output_value attribute output_value = workflow_invocation_output_value From 4db734302423c2a2228f4c8e4e7f72ce1d105a6b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 20:56:20 -0400 Subject: [PATCH 216/225] Ensure args do not include extra fixtures; format Also remove excessively long lines --- test/unit/model/test_mapping.py | 272 ++++++++++++++++++++++---------- 1 file changed, 192 insertions(+), 80 deletions(-) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index f6a183436bd..7e4e9f3e3ad 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -266,8 +266,11 @@ class TestCleanupEventMetadataFileAssociation(BaseTest): def test_columns(self, session, cls_, cleanup_event, metadata_file): create_time = datetime.now() - obj = cls_(create_time=create_time, cleanup_event_id=cleanup_event.id, metadata_file_id=metadata_file.id) - + obj = cls_( + create_time=create_time, + cleanup_event_id=cleanup_event.id, + metadata_file_id=metadata_file.id + ) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id @@ -512,7 +515,8 @@ class TestDatasetCollection(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.elements == [dataset_collection_element] - assert stored_obj.output_dataset_collections == [job_to_implicit_output_dataset_collection_association] + assert (stored_obj.output_dataset_collections + == [job_to_implicit_output_dataset_collection_association]) class TestDatasetCollectionElement(BaseTest): @@ -645,7 +649,7 @@ class TestDatasetSource(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'dataset_source' - def test_columns(self, session, cls_, dataset, dataset_source_hash): + def test_columns(self, session, cls_, dataset): source_uri, extra_files_path, transform = 'a', 'b', 'c' obj = cls_() obj.dataset = dataset @@ -1508,7 +1512,8 @@ class TestHistoryDatasetAssociation(BaseTest): assert stored_obj.update_time == update_time assert stored_obj.state == state assert stored_obj.copied_from_history_dataset_association_id == copied_from_hda.id - assert stored_obj.copied_from_library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert (stored_obj.copied_from_library_dataset_dataset_association_id + == library_dataset_dataset_association.id) assert stored_obj.name == name assert stored_obj.info == info assert stored_obj.blurb == blurb @@ -1526,7 +1531,8 @@ class TestHistoryDatasetAssociation(BaseTest): assert stored_obj.purged == purged assert stored_obj.validated_state == validated_state assert stored_obj.validated_state_message == validated_state_message - assert stored_obj.hidden_beneath_collection_instance_id == history_dataset_collection_association.id + assert (stored_obj.hidden_beneath_collection_instance_id + == history_dataset_collection_association.id) delete_from_database(session, [copied_from_hda, parent]) @@ -1581,7 +1587,8 @@ class TestHistoryDatasetAssociation(BaseTest): assert stored_obj.copied_from_history_dataset_association.id == copied_from_hda.id assert stored_obj.copied_from_library_dataset_dataset_association.id == copied_from_ldda.id assert stored_obj.extended_metadata.id == extended_metadata.id - assert stored_obj.hidden_beneath_collection_instance.id == history_dataset_collection_association.id + assert (stored_obj.hidden_beneath_collection_instance.id + == history_dataset_collection_association.id) assert stored_obj.copied_to_library_dataset_dataset_associations == [copied_to_ldda] assert stored_obj.copied_to_history_dataset_associations == [copied_to_hda] assert stored_obj.tags == [history_dataset_association_tag_association] @@ -1719,7 +1726,6 @@ class TestHistoryDatasetAssociationSubset(BaseTest): session, cls_, history_dataset_association, - history, history_dataset_association_factory, ): hda_subset = history_dataset_association_factory() @@ -1888,7 +1894,8 @@ class TestHistoryDatasetCollectionAssociation(BaseTest): assert stored_obj.tags == [history_dataset_collection_tag_association] assert stored_obj.annotations == [history_dataset_collection_annotation_association] assert stored_obj.ratings == [history_dataset_collection_rating_association] - assert stored_obj.output_dataset_collection_instances == [job_to_output_dataset_collection_association] + assert (stored_obj.output_dataset_collection_instances + == [job_to_output_dataset_collection_association]) assert stored_obj.job_state_summary # this is a view; TODO: can we test this better? assert stored_obj.hidden_dataset_instances == [history_dataset_association] @@ -2096,7 +2103,8 @@ class TestImplicitCollectionJobs(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.jobs == [implicit_collection_jobs_job_association] - assert stored_obj.history_dataset_collection_associations == [history_dataset_collection_association] + assert (stored_obj.history_dataset_collection_associations + == [history_dataset_collection_association]) assert stored_obj.workflow_invocation_step.id == workflow_invocation_step.id @@ -2485,9 +2493,12 @@ class TestJob(BaseTest): assert stored_obj.parameters == [job_parameter] assert stored_obj.input_datasets == [job_to_input_dataset_association] assert stored_obj.input_dataset_collections == [job_to_input_dataset_collection_association] - assert stored_obj.input_dataset_collection_elements == [job_to_input_dataset_collection_element_association] - assert stored_obj.output_dataset_collection_instances == [job_to_output_dataset_collection_association] - assert stored_obj.output_dataset_collections == [job_to_implicit_output_dataset_collection_association] + assert (stored_obj.input_dataset_collection_elements + == [job_to_input_dataset_collection_element_association]) + assert (stored_obj.output_dataset_collection_instances + == [job_to_output_dataset_collection_association]) + assert (stored_obj.output_dataset_collections + == [job_to_implicit_output_dataset_collection_association]) assert stored_obj.post_job_actions == [post_job_action_association] assert stored_obj.input_library_datasets == [job_to_input_library_dataset_association] assert stored_obj.output_library_datasets == [job_to_output_library_dataset_association] @@ -2499,10 +2510,12 @@ class TestJob(BaseTest): assert stored_obj.numeric_metrics == [job_metric_numeric] assert stored_obj.job == [genome_index_tool_data] assert stored_obj.interactivetool_entry_points == [interactive_tool_entry_point] - assert stored_obj.implicit_collection_jobs_association == implicit_collection_jobs_job_association + assert (stored_obj.implicit_collection_jobs_association + == implicit_collection_jobs_job_association) assert stored_obj.container == job_container_association assert stored_obj.data_manager_association == data_manager_job_association - assert stored_obj.history_dataset_collection_associations == [history_dataset_collection_association] + assert (stored_obj.history_dataset_collection_associations + == [history_dataset_collection_association]) assert stored_obj.workflow_invocation_step == workflow_invocation_step @@ -2620,7 +2633,8 @@ class TestJobExternalOutputMetadata(BaseTest): obj = cls_(job, library_dataset_dataset_association) with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert (stored_obj.library_dataset_dataset_association_id + == library_dataset_dataset_association.id) def test_relationships( self, @@ -2660,7 +2674,7 @@ class TestJobImportHistoryArchive(BaseTest): assert stored_obj.history_id == history.id assert stored_obj. archive_dir == archive_dir - def test_relationships(self, session, cls_, job, history, dataset): + def test_relationships(self, session, cls_, job, history): obj = cls_(job, history) with dbcleanup(session, obj) as obj_id: @@ -2957,7 +2971,8 @@ class TestJobToOutputDatasetCollectionAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.job.id == job.id - assert stored_obj.dataset_collection_instance.id == history_dataset_collection_association.id + assert (stored_obj.dataset_collection_instance.id + == history_dataset_collection_association.id) class TestJobToOutputLibraryDatasetAssociation(BaseTest): @@ -3039,7 +3054,7 @@ class TestLibraryDataset(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'library_dataset' - def test_columns(self, session, cls_, library_dataset_dataset_association, library_folder): + def test_columns(self, session, cls_, library_folder): order_id = 9 create_time = datetime.now() update_time = create_time + timedelta(hours=1) @@ -3091,7 +3106,8 @@ class TestLibraryDataset(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert (stored_obj.library_dataset_dataset_association.id + == library_dataset_dataset_association.id) assert stored_obj.folder.id == library_folder.id assert stored_obj.expired_datasets[0].id == ldda.id assert stored_obj.actions == [library_dataset_permission] @@ -3114,7 +3130,8 @@ class TestLibraryDatasetCollectionAnnotationAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id + assert (stored_obj.library_dataset_collection_id + == library_dataset_collection_association.id) assert stored_obj.user_id == user.id assert stored_obj.annotation == annotation @@ -3189,7 +3206,8 @@ class TestLibraryDatasetCollectionRatingAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id + assert (stored_obj.library_dataset_collection_id + == library_dataset_collection_association.id) assert stored_obj.user_id == user.id assert stored_obj.rating == rating @@ -3216,7 +3234,8 @@ class TestLibraryDatasetCollectionTagAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert stored_obj.library_dataset_collection_id == library_dataset_collection_association.id + assert (stored_obj.library_dataset_collection_id + == library_dataset_collection_association.id) assert stored_obj.tag_id == tag.id assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname @@ -3250,7 +3269,6 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): library_dataset_dataset_association_factory, extended_metadata, user, - model, ): create_time = datetime.now() state = 'a' @@ -3303,8 +3321,10 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): assert stored_obj.dataset_id == dataset.id assert stored_obj.create_time == create_time assert stored_obj.update_time >= create_time # this is sufficient for testing - assert stored_obj.copied_from_history_dataset_association_id == history_dataset_association.id - assert stored_obj.copied_from_library_dataset_dataset_association_id == copied_from_ldda.id + assert (stored_obj.copied_from_history_dataset_association_id + == history_dataset_association.id) + assert (stored_obj.copied_from_library_dataset_dataset_association_id + == copied_from_ldda.id) assert stored_obj.state == state assert stored_obj.name == name assert stored_obj.info == info @@ -3375,8 +3395,10 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.library_dataset.id == library_dataset.id assert stored_obj.dataset.id == dataset.id - assert stored_obj.copied_from_history_dataset_association.id == history_dataset_association.id - assert stored_obj.copied_from_library_dataset_dataset_association.id == copied_from_ldda.id + assert (stored_obj.copied_from_history_dataset_association.id + == history_dataset_association.id) + assert (stored_obj.copied_from_library_dataset_dataset_association.id + == copied_from_ldda.id) assert stored_obj.extended_metadata.id == extended_metadata.id assert stored_obj.user.id == user.id assert stored_obj.tags == [library_dataset_dataset_association_tag_association] @@ -3386,7 +3408,8 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): assert stored_obj.implicitly_converted_datasets == [icda] assert stored_obj.implicitly_converted_parent_datasets == [icpda] assert stored_obj.dependent_jobs == [job_to_input_library_dataset_association] - assert stored_obj.creating_job_associations == [job_to_output_library_dataset_association] + assert (stored_obj.creating_job_associations + == [job_to_output_library_dataset_association]) assert stored_obj.info_association == [library_dataset_dataset_info_association] delete_from_database(session, persisted) @@ -3411,7 +3434,8 @@ class TestLibraryDatasetDatasetAssociationPermissions(BaseTest): assert stored_obj.create_time == create_time assert stored_obj.update_time == update_time assert stored_obj.action == action - assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert (stored_obj.library_dataset_dataset_association_id + == library_dataset_dataset_association.id) assert stored_obj.role_id == role.id def test_relationships(self, session, cls_, library_dataset_dataset_association, role): @@ -3419,7 +3443,8 @@ class TestLibraryDatasetDatasetAssociationPermissions(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert (stored_obj.library_dataset_dataset_association.id + == library_dataset_dataset_association.id) assert stored_obj.role.id == role.id @@ -3437,7 +3462,8 @@ class TestLibraryDatasetDatasetAssociationTagAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert (stored_obj.library_dataset_dataset_association_id + == library_dataset_dataset_association.id) assert stored_obj.tag_id == tag.id assert stored_obj.user_id == user.id assert stored_obj.user_tname == user_tname @@ -3450,7 +3476,8 @@ class TestLibraryDatasetDatasetAssociationTagAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert (stored_obj.library_dataset_dataset_association.id + == library_dataset_dataset_association.id) assert stored_obj.tag.id == tag.id assert stored_obj.user.id == user.id @@ -3475,7 +3502,8 @@ class TestLibraryDatasetDatasetInfoAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) assert stored_obj.id == obj_id - assert stored_obj.library_dataset_dataset_association_id == library_dataset_dataset_association.id + assert (stored_obj.library_dataset_dataset_association_id + == library_dataset_dataset_association.id) assert stored_obj.form_definition_id == form_definition.id assert stored_obj.form_values_id == form_values.id assert stored_obj.deleted == deleted @@ -3492,7 +3520,8 @@ class TestLibraryDatasetDatasetInfoAssociation(BaseTest): with dbcleanup(session, obj) as obj_id: stored_obj = get_stored_obj(session, cls_, obj_id) - assert stored_obj.library_dataset_dataset_association.id == library_dataset_dataset_association.id + assert (stored_obj.library_dataset_dataset_association.id + == library_dataset_dataset_association.id) assert stored_obj.template.id == form_definition.id assert stored_obj.info.id == form_values.id @@ -4186,8 +4215,14 @@ class TestQuota(BaseTest): assert stored_obj.operation == operation assert stored_obj.deleted is False - def test_relationships(self, session, cls_, default_quota_association, group_quota_association, user_quota_association): - + def test_relationships( + self, + session, + cls_, + default_quota_association, + group_quota_association, + user_quota_association + ): def add_association(assoc_object, assoc_attribute): assoc_object.quota = obj getattr(obj, assoc_attribute).append(assoc_object) @@ -4256,7 +4291,8 @@ class TestRole(BaseTest): assert stored_obj.library_actions == [library_permission] assert stored_obj.library_folder_actions == [library_folder_permission] assert stored_obj.library_dataset_actions == [library_dataset_permission] - assert stored_obj.library_dataset_dataset_actions == [library_dataset_dataset_association_permission] + assert (stored_obj.library_dataset_dataset_actions + == [library_dataset_dataset_association_permission]) assert stored_obj.users == [user_role_association] @@ -4360,7 +4396,8 @@ class TestStoredWorkflow(BaseTest): user, stored_workflow_rating_association_factory ): - _run_average_rating_test(session, stored_workflow, user, stored_workflow_rating_association_factory) + _run_average_rating_test( + session, stored_workflow, user, stored_workflow_rating_association_factory) class TestStoredWorkflowAnnotationAssociation(BaseTest): @@ -4553,14 +4590,19 @@ class TestTag(BaseTest): getattr(obj, assoc_attribute).append(assoc_object) add_association(history_tag_association, 'tagged_histories') - add_association(history_dataset_association_tag_association, 'tagged_history_dataset_associations') - add_association(library_dataset_dataset_association_tag_association, 'tagged_library_dataset_dataset_associations') + add_association( + history_dataset_association_tag_association, 'tagged_history_dataset_associations') + add_association( + library_dataset_dataset_association_tag_association, + 'tagged_library_dataset_dataset_associations') add_association(page_tag_association, 'tagged_pages') add_association(workflow_step_tag_association, 'tagged_workflow_steps') add_association(stored_workflow_tag_association, 'tagged_stored_workflows') add_association(visualization_tag_association, 'tagged_visualizations') - add_association(history_dataset_collection_tag_association, 'tagged_history_dataset_collections') - add_association(library_dataset_collection_tag_association, 'tagged_library_dataset_collections') + add_association( + history_dataset_collection_tag_association, 'tagged_history_dataset_collections') + add_association( + library_dataset_collection_tag_association, 'tagged_library_dataset_collections') add_association(tool_tag_association, 'tagged_tools') with dbcleanup(session, obj) as obj_id: @@ -4568,14 +4610,18 @@ class TestTag(BaseTest): assert stored_obj.parent.id == parent_tag.id assert stored_obj.children == [child_tag] assert stored_obj.tagged_histories == [history_tag_association] - assert stored_obj.tagged_history_dataset_associations == [history_dataset_association_tag_association] - assert stored_obj.tagged_library_dataset_dataset_associations == [library_dataset_dataset_association_tag_association] + assert (stored_obj.tagged_history_dataset_associations + == [history_dataset_association_tag_association]) + assert (stored_obj.tagged_library_dataset_dataset_associations + == [library_dataset_dataset_association_tag_association]) assert stored_obj.tagged_pages == [page_tag_association] assert stored_obj.tagged_workflow_steps == [workflow_step_tag_association] assert stored_obj.tagged_stored_workflows == [stored_workflow_tag_association] assert stored_obj.tagged_visualizations == [visualization_tag_association] - assert stored_obj.tagged_history_dataset_collections == [history_dataset_collection_tag_association] - assert stored_obj.tagged_library_dataset_collections == [library_dataset_collection_tag_association] + assert (stored_obj.tagged_history_dataset_collections + == [history_dataset_collection_tag_association]) + assert (stored_obj.tagged_library_dataset_collections + == [library_dataset_collection_tag_association]) assert stored_obj.tagged_tools == [tool_tag_association] @@ -4917,7 +4963,8 @@ class TestUser(BaseTest): assert stored_obj.workflows_shared_by_others == [stored_workflow_user_share_association] assert stored_obj.roles == [user_role_association] assert stored_obj.stored_workflows == [stored_workflow] - assert stored_obj.visualizations_shared_by_others == [visualization_user_share_association] + assert (stored_obj.visualizations_shared_by_others + == [visualization_user_share_association]) delete_from_database(session, [history1, history2, swme]) @@ -5216,8 +5263,15 @@ class TestVisualization(BaseTest): delete_from_database(session, revision2) - def test_average_rating(self, session, visualization, user, visualization_rating_association_factory): - _run_average_rating_test(session, visualization, user, visualization_rating_association_factory) + def test_average_rating( + self, + session, + visualization, + user, + visualization_rating_association_factory + ): + _run_average_rating_test( + session, visualization, user, visualization_rating_association_factory) class TestVisualizationAnnotationAssociation(BaseTest): @@ -5504,8 +5558,10 @@ class TestWorkflowInvocation(BaseTest): workflow_request_to_input_dataset_collection_association, workflow_invocation_output_value, ): - subworkflow_invocation_assoc = workflow_invocation_to_subworkflow_invocation_association_factory() - parent_workflow_invocation_assoc = workflow_invocation_to_subworkflow_invocation_association_factory() + subworkflow_invocation_assoc = \ + workflow_invocation_to_subworkflow_invocation_association_factory() + parent_workflow_invocation_assoc = \ + workflow_invocation_to_subworkflow_invocation_association_factory() obj = cls_() obj.workflow = workflow @@ -5514,10 +5570,12 @@ class TestWorkflowInvocation(BaseTest): obj.step_states.append(workflow_request_step_state) obj.input_step_parameters.append(workflow_request_input_step_parameter) obj.input_datasets.append(workflow_request_to_input_dataset_association) - obj.input_dataset_collections.append(workflow_request_to_input_dataset_collection_association) + obj.input_dataset_collections.append( + workflow_request_to_input_dataset_collection_association) obj.subworkflow_invocations.append(subworkflow_invocation_assoc) obj.steps.append(workflow_invocation_step) - obj.output_dataset_collections.append(workflow_invocation_output_dataset_collection_association) + obj.output_dataset_collections.append( + workflow_invocation_output_dataset_collection_association) obj.output_datasets.append(workflow_invocation_output_dataset_association) obj.parent_workflow_invocation_association.append(parent_workflow_invocation_assoc) obj.output_values.append(workflow_invocation_output_value) @@ -5531,15 +5589,19 @@ class TestWorkflowInvocation(BaseTest): assert stored_obj.step_states == [workflow_request_step_state] assert stored_obj.input_step_parameters == [workflow_request_input_step_parameter] assert stored_obj.input_datasets == [workflow_request_to_input_dataset_association] - assert stored_obj.input_dataset_collections == [workflow_request_to_input_dataset_collection_association] + assert (stored_obj.input_dataset_collections + == [workflow_request_to_input_dataset_collection_association]) assert stored_obj.subworkflow_invocations == [subworkflow_invocation_assoc] assert stored_obj.steps == [workflow_invocation_step] - assert stored_obj.output_dataset_collections == [workflow_invocation_output_dataset_collection_association] + assert (stored_obj.output_dataset_collections + == [workflow_invocation_output_dataset_collection_association]) assert stored_obj.output_datasets == [workflow_invocation_output_dataset_association] - assert stored_obj.parent_workflow_invocation_association == [parent_workflow_invocation_assoc] + assert (stored_obj.parent_workflow_invocation_association + == [parent_workflow_invocation_assoc]) assert stored_obj.output_values == [workflow_invocation_output_value] - delete_from_database(session, [subworkflow_invocation_assoc, parent_workflow_invocation_assoc]) + delete_from_database( + session, [subworkflow_invocation_assoc, parent_workflow_invocation_assoc]) class TestWorkflowInvocationOutputDatasetAssociation(BaseTest): @@ -5770,7 +5832,8 @@ class TestWorkflowInvocationStep(BaseTest): obj.workflow_step = workflow_step obj.job = job obj.implicit_collection_jobs = implicit_collection_jobs - obj.output_dataset_collections.append(workflow_invocation_step_output_dataset_collection_association) + obj.output_dataset_collections.append( + workflow_invocation_step_output_dataset_collection_association) obj.output_datasets.append(workflow_invocation_step_output_dataset_association) with dbcleanup(session, obj) as obj_id: @@ -5779,8 +5842,10 @@ class TestWorkflowInvocationStep(BaseTest): assert stored_obj.workflow_step.id == workflow_step.id assert stored_obj.job.id == job.id assert stored_obj.implicit_collection_jobs.id == implicit_collection_jobs.id - assert stored_obj.output_dataset_collections == [workflow_invocation_step_output_dataset_collection_association] - assert stored_obj.output_datasets == [workflow_invocation_step_output_dataset_association] + assert (stored_obj.output_dataset_collections + == [workflow_invocation_step_output_dataset_collection_association]) + assert (stored_obj.output_datasets + == [workflow_invocation_step_output_dataset_association]) assert stored_obj.output_value.id == workflow_invocation_output_value.id def test_subworkflow_invocation_attribute( @@ -5826,7 +5891,13 @@ class TestWorkflowInvocationStepOutputDatasetAssociation(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'workflow_invocation_step_output_dataset_association' - def test_columns(self, session, cls_, workflow_invocation_step, history_dataset_association): + def test_columns( + self, + session, + cls_, + workflow_invocation_step, + history_dataset_association + ): output_name = 'a' obj = cls_() obj.workflow_invocation_step = workflow_invocation_step @@ -5840,7 +5911,13 @@ class TestWorkflowInvocationStepOutputDatasetAssociation(BaseTest): assert stored_obj.dataset_id == history_dataset_association.id assert stored_obj.output_name == output_name - def test_relationships(self, session, cls_, workflow_invocation_step, history_dataset_association): + def test_relationships( + self, + session, + cls_, + workflow_invocation_step, + history_dataset_association + ): obj = cls_() obj.workflow_invocation_step = workflow_invocation_step obj.dataset = history_dataset_association @@ -5934,7 +6011,6 @@ class TestWorkflowInvocationToSubworkflowInvocationAssociation(BaseTest): cls_, workflow_invocation, workflow_step, - workflow, workflow_invocation_factory, ): parent_workflow_invocation = workflow_invocation_factory() @@ -6068,7 +6144,14 @@ class TestWorkflowRequestToInputDatasetAssociation(BaseTest): def test_table(self, cls_): assert cls_.__tablename__ == 'workflow_request_to_input_dataset' - def test_columns(self, session, cls_, workflow_step, workflow_invocation, history_dataset_association): + def test_columns( + self, + session, + cls_, + workflow_step, + workflow_invocation, + history_dataset_association + ): name = 'a' obj = cls_() obj.name = name @@ -6083,7 +6166,14 @@ class TestWorkflowRequestToInputDatasetAssociation(BaseTest): assert stored_obj.workflow_step_id == workflow_step.id assert stored_obj.dataset_id == history_dataset_association.id - def test_relationships(self, session, cls_, workflow_step, workflow_invocation, history_dataset_association): + def test_relationships( + self, + session, + cls_, + workflow_step, + workflow_invocation, + history_dataset_association + ): obj = cls_() obj.workflow_invocation = workflow_invocation obj.workflow_step = workflow_step @@ -6449,7 +6539,7 @@ def cloud_authz(model, session, user, user_authnz_token): @pytest.fixture -def custos_authnz_token(model, session, user, user_authnz_token): +def custos_authnz_token(model, session, user): instance = model.CustosAuthnzToken(user, None, None, None, None, None, None, None) yield from dbcleanup_wrapper(session, instance) @@ -6572,7 +6662,7 @@ def form_values(model, session): @pytest.fixture -def galaxy_session(model, session, user): +def galaxy_session(model, session): instance = model.GalaxySession(session_key=get_unique_value()) yield from dbcleanup_wrapper(session, instance) @@ -6668,7 +6758,8 @@ def history_dataset_collection_rating_association( user, history_dataset_collection_association, ): - instance = model.HistoryDatasetCollectionRatingAssociation(user, history_dataset_collection_association) + instance = model.HistoryDatasetCollectionRatingAssociation( + user, history_dataset_collection_association) yield from dbcleanup_wrapper(session, instance) @@ -6719,8 +6810,13 @@ def implicitly_converted_dataset_association(model, session, history_dataset_ass @pytest.fixture -def implicitly_created_dataset_collection_input(model, session, history_dataset_collection_association): - instance = model.ImplicitlyCreatedDatasetCollectionInput(None, history_dataset_collection_association) +def implicitly_created_dataset_collection_input( + model, + session, + history_dataset_collection_association +): + instance = model.ImplicitlyCreatedDatasetCollectionInput( + None, history_dataset_collection_association) yield from dbcleanup_wrapper(session, instance) @@ -6791,8 +6887,13 @@ def job_to_input_dataset_association(model, session, history_dataset_association @pytest.fixture -def job_to_input_dataset_collection_association(model, session, history_dataset_collection_association): - instance = model.JobToInputDatasetCollectionAssociation(None, history_dataset_collection_association) +def job_to_input_dataset_collection_association( + model, + session, + history_dataset_collection_association +): + instance = model.JobToInputDatasetCollectionAssociation( + None, history_dataset_collection_association) yield from dbcleanup_wrapper(session, instance) @@ -6815,8 +6916,13 @@ def job_to_output_dataset_association(model, session, history_dataset_associatio @pytest.fixture -def job_to_output_dataset_collection_association(model, session, history_dataset_collection_association): - instance = model.JobToOutputDatasetCollectionAssociation(None, history_dataset_collection_association) +def job_to_output_dataset_collection_association( + model, + session, + history_dataset_collection_association +): + instance = model.JobToOutputDatasetCollectionAssociation( + None, history_dataset_collection_association) yield from dbcleanup_wrapper(session, instance) @@ -6875,8 +6981,14 @@ def library_dataset_dataset_association(model, session): @pytest.fixture -def library_dataset_dataset_association_permission(model, session, library_dataset_dataset_association, role): - instance = model.LibraryDatasetDatasetAssociationPermissions('a', library_dataset_dataset_association, role) +def library_dataset_dataset_association_permission( + model, + session, + library_dataset_dataset_association, + role +): + instance = model.LibraryDatasetDatasetAssociationPermissions( + 'a', library_dataset_dataset_association, role) yield from dbcleanup_wrapper(session, instance) @@ -7085,7 +7197,7 @@ def user(model, session): @pytest.fixture -def user_address(model, session, user): +def user_address(model, session): instance = model.UserAddress() instance.name = 'a' instance.address = 'b' @@ -7316,14 +7428,14 @@ def history_factory(model): @pytest.fixture -def history_rating_association_factory(model, user, history): +def history_rating_association_factory(model): def make_instance(*args, **kwds): return model.HistoryRatingAssociation(*args, **kwds) return make_instance @pytest.fixture -def implicitly_converted_dataset_association_factory(model, user, history_dataset_association): +def implicitly_converted_dataset_association_factory(model, history_dataset_association): def make_instance(*args, **kwds): instance = model.ImplicitlyConvertedDatasetAssociation( dataset=history_dataset_association, From abbc256f810390b57a4742cc53a10f52a44f885c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 16 Aug 2021 22:49:04 -0400 Subject: [PATCH 217/225] Add util function to test for entity equality in collections Occasionally we need to compare two sets of model instances: expected and retrtieved from the database. The objects are not the same, so we can't compare them directly. We only care about primary key equality. We also care about the quantity of each instance in each set, so {a, a, b} != {a, b, b} --- test/unit/model/test_mapping.py | 36 +++++++++++++++++++--------- test/unit/model/test_test_mapping.py | 21 ++++++++++++++++ 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 7e4e9f3e3ad..97cfaa6d260 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -463,8 +463,8 @@ class TestDataset(BaseTest): assert stored_obj.hashes == [dataset_hash] assert stored_obj.sources == [dataset_source] assert stored_obj.library_associations == [library_dataset_dataset_association] - assert len(stored_obj.history_associations) == 2 - # TODO: test membership in history_associations by id + assert are_same_entity_collections( + stored_obj.history_associations, [hda, history_dataset_association]) delete_from_database(session, hda) @@ -4380,10 +4380,8 @@ class TestStoredWorkflow(BaseTest): assert stored_obj.ratings == [stored_workflow_rating_association] # This doesn't test the average amount, just the mapping. assert stored_obj.average_rating == stored_workflow_rating_association.rating - - assert len(stored_obj.tags) == 2 - assert stored_workflow_tag_association in stored_obj.tags - assert tag_assoc2 in stored_obj.tags + assert are_same_entity_collections( + stored_obj.tags, [stored_workflow_tag_association, tag_assoc2]) assert stored_obj.owner_tags == [tag_assoc2] assert stored_obj.users_shared_with == [stored_workflow_user_share_association] @@ -4943,11 +4941,7 @@ class TestUser(BaseTest): assert stored_obj.custos_auth == [custos_authnz_token] assert stored_obj.default_permissions == [default_user_permissions] assert stored_obj.groups == [user_group_association] - - assert len(stored_obj.histories) == 2 - assert history1 in stored_obj.histories - assert history2 in stored_obj.histories - + assert are_same_entity_collections(stored_obj.histories, [history1, history2]) assert stored_obj.active_histories == [history1] assert stored_obj.galaxy_sessions == [galaxy_session] assert stored_obj.pages_shared_by_others == [page_user_share_association] @@ -7638,6 +7632,26 @@ def get_unique_value(): return uuid4().hex +def are_same_entity_collections(collection1, collection2): + """ + The 2 arguments are collections of instances of models that have an `id` + attribute as their primary key. + Returns `True` if collections are the same size and contain the same + instances. Instance equality is determined by the object's `id` attribute, + not its Python object identifier. + """ + if len(collection1) != len(collection2): + return False + + collection1.sort(key=lambda item: item.id) + collection2.sort(key=lambda item: item.id) + + for item1, item2 in zip(collection1, collection2): + if item1.id != item2.id: + return False + return True + + def _run_average_rating_test(session, obj, user, obj_rating_association_factory): # obj has been expunged; to access its deferred properties, # it needs to be added back to the session. diff --git a/test/unit/model/test_test_mapping.py b/test/unit/model/test_test_mapping.py index eca626f6b4e..14dc653b017 100644 --- a/test/unit/model/test_test_mapping.py +++ b/test/unit/model/test_test_mapping.py @@ -17,6 +17,7 @@ from sqlalchemy.exc import NoResultFound from sqlalchemy.orm import registry, Session from . test_mapping import ( + are_same_entity_collections, dbcleanup, dbcleanup_wrapper, delete_from_database, @@ -160,6 +161,26 @@ def test_has_unique_constraint(session): assert not has_unique_constraint(Foo.__table__, ('field1',)) +def test_are_same_entity_collections(session): + foo1 = Foo() + foo2 = Foo() + foo3 = Foo() + persist(session, foo1) + persist(session, foo2) + persist(session, foo3) + + stored_foo1 = _get_stored_instance_by_id(session, Foo, foo1.id) + stored_foo2 = _get_stored_instance_by_id(session, Foo, foo2.id) + stored_foo3 = _get_stored_instance_by_id(session, Foo, foo3.id) + + expected = [foo1, foo2] + + assert are_same_entity_collections([stored_foo1, stored_foo2], expected) + assert are_same_entity_collections([stored_foo2, stored_foo1], expected) + assert not are_same_entity_collections([stored_foo1, stored_foo3], expected) + assert not are_same_entity_collections([stored_foo1, stored_foo1, stored_foo2], expected) + + # Test utilities mapper_registry = registry() From fc717fdb22fc5d8aabd764e86e03ec60d4d5bda8 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 17 Aug 2021 00:40:43 -0400 Subject: [PATCH 218/225] Expunge deleted obj before creating another w/same id Gets rid of SAWarning (see comment for details) --- test/unit/model/test_mapping.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index 97cfaa6d260..ed09e54a83e 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -2629,6 +2629,10 @@ class TestJobExternalOutputMetadata(BaseTest): assert stored_obj.filename_override_metadata == filename_override_metadata assert stored_obj.job_runner_external_pid == job_runner_external_pid + # Expunge from session: we're creating and storing a new object that will recieve + # the same primary key. This will trigger a SAWarning, since this identity already + # exists in SQLAlchemy's identity map. + session.expunge(stored_obj) # Now pass an ldda (w/no extra fields, since we've just tested them) obj = cls_(job, library_dataset_dataset_association) with dbcleanup(session, obj) as obj_id: @@ -2651,6 +2655,8 @@ class TestJobExternalOutputMetadata(BaseTest): assert stored_obj.job.id == job.id assert stored_obj.dataset.id == history_dataset_association.id + # Expunge: see comment in test_columns() + session.expunge(stored_obj) # Now pass an ldda obj = cls_(job, library_dataset_dataset_association) with dbcleanup(session, obj) as obj_id: From a7cc1df721504393be23675b7551c4cafeda24bf Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 17 Aug 2021 21:02:06 -0400 Subject: [PATCH 219/225] Edit documentation at the top of the module --- test/unit/model/test_mapping.py | 68 +++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/test/unit/model/test_mapping.py b/test/unit/model/test_mapping.py index ed09e54a83e..831fd5544f4 100644 --- a/test/unit/model/test_mapping.py +++ b/test/unit/model/test_mapping.py @@ -1,45 +1,63 @@ """ -TODO: documentaion on writing model tests goes here +This module contains tests for column- and relationship-mapped attributes of +models, table names, as well as table-level indexes and constraints. These +tests do not address any business logic defined in the model. -Example: +Tests are grouped into classes, one for each model (tests do NOT share class instances: +https://docs.pytest.org/en/6.2.x/getting-started.html#group-multiple-tests-in-a-class) -TestFoo(BaseTest): # BaseTest is parent class - # test table-level properties (e.g. table name, indexes, unique constraints) - def test_table(self, cls_): # cls_ is fixture defined in parent class, returns class under test. - assert cls_.__tablename__ == 'foo' # assert table name - assert has_index(cls.__table__, ('foo',)) # second arg is a tuple containg field names +Example of a simple model and its tests: + +class Planet(Base): + __tablename__ = 'planet' + __table_args__ = (Index('index_name', 'name'),) + + id = Column(Integer, primary_key=True) + name = Column(String) + star_id = Column(Integer, ForeignKey('star.id')) + + star = relationship('Star') + satellites = relationship('Satellite') + + def __init__(self, name): + self.name = name + + +class TestPlanet(BaseTest): # BaseTest is a base class; we need it to get the type of the model under test + + # test table name and table-level indexes and constraints + def test_table(self, cls_): # cls_ is a fixture defined in BaseTest, returns the type under test. + assert cls_.__tablename__ == 'planet' # verify table name + assert has_index(cls.__table__, ('name',)) # verify index; second arg is a tuple containg field names # test column-mapped fields - def test_columns(self, session, cls_): - some_foo, some_bar = 42, 'stuff' # create test values here - obj = cls_(foo=some_foo) # pass test values to constructor - obj.bar = some_bar # assign test values to obj if can't pass to constructor + def test_columns(self, session, cls_, star): # star is a fixture that provides a persisted instance of Star + name = 'Saturn' # create test values + obj = cls_(name=name) # pass test values to constructor + obj.star = star # assign test values to obj if can't pass to constructor with dbcleanup(session, obj) as obj_id: # use context manager to ensure obj is deleted from db on exit. stored_obj = get_stored_obj(session, cls_, obj_id) # retrieve data from db and create new obj. # check ALL column-mapped fields assert stored_obj.id == obj_id - assert stored_obj.create_time == create_time - assert stored_obj.user_id == user_id - assert stored_obj.key == key + assert stored_obj.name == name + assert stored_obj.star_id == star.id # test the column attribute star_id # test relationship-mapped fields - def test_relationships(self, session, cls_): - obj = cls_() # use minimal possible constructor + def test_relationships(self, session, cls_, star, satellite): # satellite is a fixture for Satellite + obj = cls_(name=name) # use minimal possible constructor + obj.star = star # assign test values to test relationship-mapped attributes (not passed to constructor) + obj.satellites.append(satellite) # add a related object - with dbcleanup(session, obj) as obj_id: + with dbcleanup(session, obj) as obj_id: # same as in previous test: store, then retrieve stored_obj = get_stored_obj(session, cls_, obj_id) # check ALL relationship-mapped fields + assert stored_obj.star.id == star.id # test the relationship attribute star.id (note the dot operator) + assert stored_obj.satellites == [satellite] # verify collecion of related objects -# TODO: ADD MORE EXAMPLES! -- point to TestLibraryDataset for example of using more than one object of same type (and, thus, - not being able to use a fixture. -- TestPage: how to call cleanup explicitly (see average_ratings) -# TODO: explain why we test for columns: - assert stored_obj.user_id == user_id - and for relationships: - assert stored_obj.user.id == user_id + +See other model tests in this module for examples of more complex setups. """ from contextlib import contextmanager From f8f691434463accf1dea15db6324e24f79490bb4 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 3 Sep 2021 15:17:06 -0400 Subject: [PATCH 220/225] Do not set SQLAlchemy state on a wrapped object; undo 0e3afe7 Here's what happens when an instance of SafeStringWrapper is created. (self = instance of SafeStringWrapper) A class mapped declaratively is a subclass of DeclarativeMeta. It will check at creation time if self has _sa_instance_state set, and if not, it'll try to set it. This happens BEFORE self.__init__ has been called, so self.unsanitized does not exists, which raises an AttributeError. To avoid this, as well as to avoid SQLAlchemy state to be set on SafeStringWrapper, we simply ignore this call. --- lib/galaxy/tools/evaluation.py | 10 ---------- lib/galaxy/util/object_wrapper.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 9c8df5a795e..fa231967fa9 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -4,8 +4,6 @@ import os import shlex import tempfile -import sqlalchemy - from galaxy import model from galaxy.files import ProvidesUserFileSourcesUserContext from galaxy.job_execution.setup import ensure_configs_directory @@ -429,14 +427,6 @@ class ToolEvaluator: skip = ['chromInfo'] + list(self.tool.template_macro_params.keys()) if not self.tool or not self.tool.options or self.tool.options.sanitize: for key, value in list(param_dict.items()): - - # Do not wrap Galaxy model classes with SafeStringWrapper. - # Models are mapped declaratively, so SQLAlchemy's DeclarativeMeta is their metaclass. - # By overriding attribute access, SafeStringWrapper (called in wrap_with_safe_string) - # inteferes with SQLAlchemy's instrumentation. And everything dies. - if type(value.__class__) == sqlalchemy.orm.decl_api.DeclarativeMeta: - continue - if key not in skip: # Remove key so that new wrapped object will occupy key slot del param_dict[key] diff --git a/lib/galaxy/util/object_wrapper.py b/lib/galaxy/util/object_wrapper.py index 16b71f599f3..bec7d268710 100644 --- a/lib/galaxy/util/object_wrapper.py +++ b/lib/galaxy/util/object_wrapper.py @@ -23,6 +23,8 @@ from types import ( TracebackType, ) +import sqlalchemy + NoneType = type(None) NotImplementedType = type(NotImplemented) EllipsisType = type(Ellipsis) @@ -261,6 +263,14 @@ class SafeStringWrapper: return self.__safe_string_wrapper_function__(getattr(self.unsanitized, name)) def __setattr__(self, name, value): + # A class mapped declaratively is a subclass of DeclarativeMeta. It will check at creation time + # if self has _sa_instance_state set, and if not, it'll try to set it. This happens BEFORE self.__init__ + # has been called, so self.unsanitized does not exists, which raises an AttributeError. + # To avoid this, as well as to avoid SQLAlchemy state to be set on SafeStringWrapper, + # we simply ignore this call. + if isinstance(value, sqlalchemy.orm.state.InstanceState): + return + if name in SafeStringWrapper.__NO_WRAP_NAMES__: return object.__setattr__(self, name, value) return setattr(self.unsanitized, name, value) From ac93e0836799ac7d9a7637f58abc3bdb929b9bac Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 3 Sep 2021 15:42:03 -0400 Subject: [PATCH 221/225] Add SQLAlchemy to util package requirements We need this is util/object_wrapper --- packages/util/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index 47b453fb12f..3000635b9bc 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -7,4 +7,5 @@ pyyaml requests routes six>=1.9.0 +SQLAlchemy>=1.4.20,<2 zipstream-new From 548e3de7647f2ab4642023e2958cbc67f66160cd Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 6 Sep 2021 10:46:16 -0400 Subject: [PATCH 222/225] Update packages/util/requirements.txt Co-authored-by: Marius van den Beek --- packages/util/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index 3000635b9bc..47b453fb12f 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -7,5 +7,4 @@ pyyaml requests routes six>=1.9.0 -SQLAlchemy>=1.4.20,<2 zipstream-new From a6ab79a95df7da8c507241c900c9d95fdbe4b2ec Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 6 Sep 2021 11:16:24 -0400 Subject: [PATCH 223/225] Move object_wrapper to security package For context see discussion in #12064 --- lib/galaxy/model/metadata.py | 2 +- lib/galaxy/{util => security}/object_wrapper.py | 0 lib/galaxy/tools/evaluation.py | 2 +- lib/galaxy/tools/wrappers.py | 2 +- lib/galaxy/util/dbkeys.py | 2 +- packages/util/galaxy/security | 1 + packages/util/requirements.txt | 2 ++ 7 files changed, 7 insertions(+), 4 deletions(-) rename lib/galaxy/{util => security}/object_wrapper.py (100%) create mode 120000 packages/util/galaxy/security diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index ea504a0aba8..9d448d6df36 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -18,6 +18,7 @@ from sqlalchemy.orm import object_session from sqlalchemy.orm.attributes import flag_modified import galaxy.model +from galaxy.security.object_wrapper import sanitize_lists_to_string from galaxy.util import ( form_builder, listify, @@ -26,7 +27,6 @@ from galaxy.util import ( unicodify, ) from galaxy.util.json import safe_dumps -from galaxy.util.object_wrapper import sanitize_lists_to_string log = logging.getLogger(__name__) diff --git a/lib/galaxy/util/object_wrapper.py b/lib/galaxy/security/object_wrapper.py similarity index 100% rename from lib/galaxy/util/object_wrapper.py rename to lib/galaxy/security/object_wrapper.py diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index fa231967fa9..5e2e748155c 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -8,6 +8,7 @@ from galaxy import model from galaxy.files import ProvidesUserFileSourcesUserContext from galaxy.job_execution.setup import ensure_configs_directory from galaxy.model.none_like import NoneDataset +from galaxy.security.object_wrapper import wrap_with_safe_string from galaxy.tools import global_tool_errors from galaxy.tools.parameters import ( visit_input_values, @@ -41,7 +42,6 @@ from galaxy.util import ( unicodify, ) from galaxy.util.bunch import Bunch -from galaxy.util.object_wrapper import wrap_with_safe_string from galaxy.util.template import fill_template from galaxy.work.context import WorkRequestContext diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py index 5da9940b158..414c0a4ae5a 100644 --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -6,12 +6,12 @@ from functools import total_ordering from galaxy import exceptions from galaxy.model.none_like import NoneDataset +from galaxy.security.object_wrapper import wrap_with_safe_string from galaxy.tools.parameters.wrapped_json import ( data_collection_input_to_staging_path_and_source_path, data_input_to_staging_path_and_source_path, ) from galaxy.util import filesystem_safe_string -from galaxy.util.object_wrapper import wrap_with_safe_string log = logging.getLogger(__name__) diff --git a/lib/galaxy/util/dbkeys.py b/lib/galaxy/util/dbkeys.py index c29daf574c3..2a4762e8192 100644 --- a/lib/galaxy/util/dbkeys.py +++ b/lib/galaxy/util/dbkeys.py @@ -6,11 +6,11 @@ import os.path import re from json import loads +from galaxy.security.object_wrapper import sanitize_lists_to_string from galaxy.util import ( galaxy_directory, unicodify, ) -from galaxy.util.object_wrapper import sanitize_lists_to_string log = logging.getLogger(__name__) diff --git a/packages/util/galaxy/security b/packages/util/galaxy/security new file mode 120000 index 00000000000..765c8ef7770 --- /dev/null +++ b/packages/util/galaxy/security @@ -0,0 +1 @@ +../../../lib/galaxy/security/ \ No newline at end of file diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index 47b453fb12f..14ed2fb05f9 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -3,8 +3,10 @@ boltons docutils markupsafe packaging +pycryptodome pyyaml requests routes six>=1.9.0 +SQLAlchemy>=1.4.20,<2 zipstream-new From 73d87f5d3360ea7bd96467813d404217decde95f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 7 Sep 2021 09:47:35 -0400 Subject: [PATCH 224/225] Update packages/util/requirements.txt Co-authored-by: Marius van den Beek --- packages/util/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index 14ed2fb05f9..4c0db6e879a 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -8,5 +8,4 @@ pyyaml requests routes six>=1.9.0 -SQLAlchemy>=1.4.20,<2 zipstream-new From da7fdddd2b57ba5a56ad8fe968daaf3d1336d2b3 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Tue, 7 Sep 2021 13:13:04 -0400 Subject: [PATCH 225/225] Remove references to security from util package --- lib/galaxy/util/dbkeys.py | 2 +- packages/util/galaxy/security | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 120000 packages/util/galaxy/security diff --git a/lib/galaxy/util/dbkeys.py b/lib/galaxy/util/dbkeys.py index 2a4762e8192..09525da1ade 100644 --- a/lib/galaxy/util/dbkeys.py +++ b/lib/galaxy/util/dbkeys.py @@ -6,9 +6,9 @@ import os.path import re from json import loads -from galaxy.security.object_wrapper import sanitize_lists_to_string from galaxy.util import ( galaxy_directory, + sanitize_lists_to_string, unicodify, ) diff --git a/packages/util/galaxy/security b/packages/util/galaxy/security deleted file mode 120000 index 765c8ef7770..00000000000 --- a/packages/util/galaxy/security +++ /dev/null @@ -1 +0,0 @@ -../../../lib/galaxy/security/ \ No newline at end of file