From 67977bc394b5d7cb54c29ea73e0bcb323e571d0f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 13 Oct 2021 14:48:34 -0400 Subject: [PATCH 1/9] Restructure HDCJSS view in model/view --- lib/galaxy/model/view/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/model/view/__init__.py b/lib/galaxy/model/view/__init__.py index 7d53206e372..f0186699596 100644 --- a/lib/galaxy/model/view/__init__.py +++ b/lib/galaxy/model/view/__init__.py @@ -7,7 +7,11 @@ from sqlalchemy.sql import column, text from galaxy.model.view.utils import View -AGGREGATE_STATE_QUERY = """ + +class HistoryDatasetCollectionJobStateSummary(View): + name = 'collection_job_state_summary_view' + + aggregate_state_query = """ SELECT hdca_id, SUM(CASE WHEN state = 'new' THEN 1 ELSE 0 END) AS new, @@ -43,11 +47,7 @@ FROM ( GROUP BY jobstates.hdca_id """ - -class HistoryDatasetCollectionJobStateSummary(View): - name = 'collection_job_state_summary_view' - - __view__ = text(AGGREGATE_STATE_QUERY).columns( + __view__ = text(aggregate_state_query).columns( column('hdca_id', Integer), column('new', Integer), column('resubmitted', Integer), From effe59a60fd7b779a8171ac7b5ef3c3ed56135ec Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 13 Oct 2021 14:59:22 -0400 Subject: [PATCH 2/9] Fix RemovedIn20Warning for views (see note) Why we are using a separate registry and not the one containing all models: see inline comment in parent View class) lib/galaxy/model/view/__init__.py:70 /home/sergey/0dev/galaxy/_galaxy/dev/lib/galaxy/model/view/__init__.py:70: RemovedIn20Warning: Calling the mapper() function directly outside of a declarative registry is deprecated. Please use the sqlalchemy.orm.registry.map_imperatively() function for a classical mapping. (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) mapper(HistoryDatasetCollectionJobStateSummary, HistoryDatasetCollectionJobStateSummary.__table__) --- lib/galaxy/model/view/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/view/__init__.py b/lib/galaxy/model/view/__init__.py index f0186699596..5d0bd48a721 100644 --- a/lib/galaxy/model/view/__init__.py +++ b/lib/galaxy/model/view/__init__.py @@ -2,7 +2,7 @@ Galaxy sql view models """ from sqlalchemy import Integer -from sqlalchemy.orm import mapper +from sqlalchemy.orm import registry from sqlalchemy.sql import column, text from galaxy.model.view.utils import View @@ -67,4 +67,6 @@ GROUP BY jobstates.hdca_id __table__ = View._make_table(name, __view__, pkeys) -mapper(HistoryDatasetCollectionJobStateSummary, HistoryDatasetCollectionJobStateSummary.__table__) +mapper_registry = registry() +mapper_registry.map_imperatively( + HistoryDatasetCollectionJobStateSummary, HistoryDatasetCollectionJobStateSummary.__table__) From 99e0c5cf75c0016f5f7cccf39937d01ac1233323 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 13 Oct 2021 15:06:16 -0400 Subject: [PATCH 3/9] Fix RemovedIn20Warning: use method chaining in select (see note) lib/galaxy/model/__init__.py:8937 /home/sergey/0dev/galaxy/_galaxy/dev/lib/galaxy/model/__init__.py:8937: RemovedIn20Warning: The legacy calling style of select() is deprecated and will be removed in SQLAlchemy 2.0. Please use the new calling style described at select(). (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) exists([HistoryDatasetCollectionAssociation.id], and_( lib/galaxy/model/__init__.py:8945 /home/sergey/0dev/galaxy/_galaxy/dev/lib/galaxy/model/__init__.py:8945: RemovedIn20Warning: The legacy calling style of select() is deprecated and will be removed in SQLAlchemy 2.0. Please use the new calling style described at select(). (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) exists([HistoryDatasetAssociation], and_( Ref: https://docs.sqlalchemy.org/en/14/errors.html#select-construct-created-in-legacy-mode-keyword-arguments-etc --- lib/galaxy/model/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fcfdc0cea22..075055ce987 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -9025,7 +9025,7 @@ mapper_registry.map_imperatively( # The following statements must not precede the mapped models defined above. Job.any_output_dataset_collection_instances_deleted = column_property( - exists([HistoryDatasetCollectionAssociation.id], and_( + exists(HistoryDatasetCollectionAssociation.id).where(and_( Job.id == JobToOutputDatasetCollectionAssociation.job_id, HistoryDatasetCollectionAssociation.id == JobToOutputDatasetCollectionAssociation.dataset_collection_id, HistoryDatasetCollectionAssociation.deleted == true()) @@ -9033,7 +9033,7 @@ Job.any_output_dataset_collection_instances_deleted = column_property( ) Job.any_output_dataset_deleted = column_property( - exists([HistoryDatasetAssociation], and_( + exists(HistoryDatasetAssociation).where(and_( Job.id == JobToOutputDatasetAssociation.job_id, HistoryDatasetAssociation.table.c.id == JobToOutputDatasetAssociation.dataset_id, HistoryDatasetAssociation.table.c.deleted == true()) From daeb6cd42b9e0ca5e184b009652a618132d7d38c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 13 Oct 2021 17:03:47 -0400 Subject: [PATCH 4/9] Fix RemovedIn20Warning: engine.execute() (note) test/unit/data/model/test_mapping.py::TestCleanupEvent::test_table /home/sergey/0dev/galaxy/_galaxy/dev/lib/galaxy/model/view/utils.py:80: RemovedIn20Warning: The Engine.execute() method is considered legacy as of the 1.x series of SQLAlchemy and will be removed in 2.0. All statement execution in SQLAlchemy 2.0 is performed by the Connection.execute() method of Connection, or in the ORM by the Session.execute() method of Session. (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) engine.execute(CreateView(view.name, view.__view__)) https://docs.sqlalchemy.org/en/14/changelog/migration_20.html#execute-method-more-strict-execution-options-are-more-prominent --- lib/galaxy/model/migrate/versions/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/model/migrate/versions/util.py b/lib/galaxy/model/migrate/versions/util.py index 1e6e8f478cc..6213cea7d6d 100644 --- a/lib/galaxy/model/migrate/versions/util.py +++ b/lib/galaxy/model/migrate/versions/util.py @@ -200,4 +200,5 @@ def execute_statements(engine, raw_sql): statements = raw_sql if isinstance(raw_sql, list) else [raw_sql] for sql in statements: cmd = DDL(sql) - cmd.execute(bind=engine) + with engine.connect() as connection: + connection.execute(cmd) From 0dc8e3f30d71211b6e906813532b19d35da3fe9b Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 13 Oct 2021 17:06:34 -0400 Subject: [PATCH 5/9] Fix RemoveIn20Warning: do not autocommit (note) test/unit/data/model/test_mapping.py: 12 warnings /home/sergey/0dev/galaxy/_galaxy/dev/lib/galaxy/model/migrate/versions/util.py:204: RemovedIn20Warning: The current statement is being autocommitted using implicit autocommit, which will be removed in SQLAlchemy 2.0. Use the .begin() method of Engine or Connection in order to use an explicit transaction for DML and DDL statements. (Background on SQLAlchemy 2.0 at: https://sqlalche.me/e/b8d9) connection.execute(cmd) --- lib/galaxy/model/migrate/versions/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/model/migrate/versions/util.py b/lib/galaxy/model/migrate/versions/util.py index 6213cea7d6d..f61ced8a927 100644 --- a/lib/galaxy/model/migrate/versions/util.py +++ b/lib/galaxy/model/migrate/versions/util.py @@ -201,4 +201,5 @@ def execute_statements(engine, raw_sql): for sql in statements: cmd = DDL(sql) with engine.connect() as connection: - connection.execute(cmd) + with connection.begin(): + connection.execute(cmd) From ec455b08a8d81d16164a78a2d38635b538027f33 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 12 Jan 2022 16:12:16 -0500 Subject: [PATCH 6/9] Fix RemoveIn20 warning: use engine.begin for explicit transaction --- test/unit/data/model/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/data/model/test_views.py b/test/unit/data/model/test_views.py index 990b0c6c6db..7357a27fc54 100644 --- a/test/unit/data/model/test_views.py +++ b/test/unit/data/model/test_views.py @@ -82,7 +82,7 @@ def make_table(metadata): def run_view_test(url, metadata, view, query): with sqlalchemy_engine(url) as engine: - with engine.connect() as conn: + with engine.begin() as conn: metadata.create_all(conn) # create table in database conn.execute(CreateView(view.name, view.__view__)) # create view in database result = conn.execute(query).fetchall() From b613c743844b4e77b5bf8c01d1ba656c8538480d Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 12 Jan 2022 16:32:02 -0500 Subject: [PATCH 7/9] Fix RemoveIn20Warning: use test() to wrap raw sql --- test/unit/data/model/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/data/model/test_views.py b/test/unit/data/model/test_views.py index 7357a27fc54..27203dddcd7 100644 --- a/test/unit/data/model/test_views.py +++ b/test/unit/data/model/test_views.py @@ -85,5 +85,5 @@ def run_view_test(url, metadata, view, query): with engine.begin() as conn: metadata.create_all(conn) # create table in database conn.execute(CreateView(view.name, view.__view__)) # create view in database - result = conn.execute(query).fetchall() + result = conn.execute(text(query)).fetchall() assert len(result) == 1 # assert that view exists in database From 74cc259bc0e0e63ddb856cf983f53607788a5443 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 12 Jan 2022 16:40:18 -0500 Subject: [PATCH 8/9] Fix RemovedIn20Warning: engine.execute --- lib/galaxy/model/view/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/view/utils.py b/lib/galaxy/model/view/utils.py index 80dc811e894..e5e0ac01804 100644 --- a/lib/galaxy/model/view/utils.py +++ b/lib/galaxy/model/view/utils.py @@ -76,5 +76,6 @@ def install_views(engine): # it mocks the app and CreateView will attempt to rebuild an existing # view in a database that is already made, the right answer is probably # to change the sql that gest emitted when CreateView is rendered. - engine.execute(DropView(view.name)) - engine.execute(CreateView(view.name, view.__view__)) + with engine.begin() as conn: + conn.execute(DropView(view.name)) + conn.execute(CreateView(view.name, view.__view__)) From df6cf535a2a245c35b1a432a2974a87ad0feed88 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 12 Jan 2022 17:02:33 -0500 Subject: [PATCH 9/9] Fix RemoveIn20Warning: Query.join(str) RemovedIn20Warning: Using strings to indicate relationship names in Query.join() is deprecated and will be removed in SQLAlchemy 2.0. Please use the class-bound attribute directly. --- lib/galaxy/model/store/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py index 9076efd16b2..724bea5b2a1 100644 --- a/lib/galaxy/model/store/__init__.py +++ b/lib/galaxy/model/store/__init__.py @@ -1394,7 +1394,7 @@ class DirectoryModelExportStore(ModelExportStore): # Write datasets' attributes to file. query = (sa_session.query(model.HistoryDatasetAssociation) .filter(model.HistoryDatasetAssociation.history == history) - .join("dataset") + .join(model.Dataset) .options(joinedload("dataset").joinedload("actions")) .order_by(model.HistoryDatasetAssociation.hid) .filter(model.Dataset.purged == expression.false()))