From bd42c761200bb43047ed65d5e851764181f51d4d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 8 Feb 2021 19:52:32 +0100 Subject: [PATCH] Fix coercion error by switching to sqlalchemy-json I was going back and forth between sqlalchemy-json and sqlalchemy-mutable. sqlalchemy-mutable is more lenient with JSON primitive values being set on a mutable field, so I preferred that. However we're seeing ``` Traceback (most recent call last): File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/__init__.py", line 1649, in handle_single_execution rval = self.execute( File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/__init__.py", line 1747, in execute return self.tool_action.execute(self, trans, incoming=incoming, set_output_hid=set_output_hid, history=history, **kwargs) File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/__init__.py", line 529, in execute handle_output(name, output) File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/__init__.py", line 442, in handle_output data.init_meta(copy_from=metadata_source) File "/Users/mvandenb/src/galaxy/lib/galaxy/model/__init__.py", line 2897, in init_meta return self.datatype.init_meta(self, copy_from=copy_from) File "/Users/mvandenb/src/galaxy/lib/galaxy/datatypes/data.py", line 181, in init_meta dataset.metadata = copy_from.metadata File "/Users/mvandenb/src/galaxy/lib/galaxy/model/__init__.py", line 2808, in set_metadata self._metadata = self.metadata.make_dict_copy(bunch) File "/Users/mvandenb/src/galaxy/.venv/lib/python3.8/site-packages/sqlalchemy/orm/attributes.py", line 272, in __set__ self.impl.set( File "/Users/mvandenb/src/galaxy/.venv/lib/python3.8/site-packages/sqlalchemy/orm/attributes.py", line 865, in set value = self.fire_replace_event( File "/Users/mvandenb/src/galaxy/.venv/lib/python3.8/site-packages/sqlalchemy/orm/attributes.py", line 873, in fire_replace_event value = fn( File "/Users/mvandenb/src/galaxy/.venv/lib/python3.8/site-packages/sqlalchemy/ext/mutable.py", line 483, in set_ value = cls.coerce(key, value) File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 280, in coerce converted_obj = cls._convert(obj) File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 317, in _convert return tracked_type(obj, root) File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable_dict.py", line 80, in __init__ super().__init__(self._convert_mapping(source)) File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 342, in _convert_mapping return { File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 343, in key: self._convert_item(item) for key, item in mapping.items() File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 334, in _convert_item return self._convert(item, self.root) File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 319, in _convert obj.root = root File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 431, in __setattr__ return super().__setattr__(name, obj) File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 390, in root for child in self._tracked_children: File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 398, in _tracked_children self.__getattribute__(name) for name in self._tracked_attr_names AttributeError: 'MutableList' object has no attribute '_tracked_attr_names' ``` with jobs that are started on before 20.09 and that finish on 21.01. sqlalchemy-json doesn't have that problem. --- .../dependencies/pipfiles/default/Pipfile | 2 +- .../pipfiles/default/pinned-requirements.txt | 2 +- lib/galaxy/model/custom_types.py | 26 +++++++------------ lib/galaxy/model/mapping.py | 16 ++++++------ .../migrate/versions/0001_initial_tables.py | 2 +- .../versions/0003_security_and_libraries.py | 6 ++--- .../migrate/versions/0008_galaxy_forms.py | 4 +-- .../versions/0019_request_library_folder.py | 2 +- .../migrate/versions/0037_samples_library.py | 4 +-- .../migrate/versions/0057_request_notify.py | 2 +- .../versions/0067_populate_sequencer_table.py | 2 +- ..._add_tool_shed_repository_table_columns.py | 2 +- ...story_dataset_association_history_table.py | 2 +- .../migrate/versions/0149_dynamic_tools.py | 2 +- packages/data/requirements.txt | 2 +- 15 files changed, 35 insertions(+), 41 deletions(-) diff --git a/lib/galaxy/dependencies/pipfiles/default/Pipfile b/lib/galaxy/dependencies/pipfiles/default/Pipfile index 9bbabc9700b..36c4c6fc3fe 100644 --- a/lib/galaxy/dependencies/pipfiles/default/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/default/Pipfile @@ -76,8 +76,8 @@ boto = "*" kombu = "*" psutil = "*" pulsar-galaxy-lib = "==0.14.1" +sqlalchemy-json = "*" sqlalchemy-migrate = "*" -sqlalchemy-mutable = "*" sqlitedict = "*" sqlparse = "*" svgwrite = "*" diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt index 68de706b27a..f596d8d628c 100644 --- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt @@ -164,8 +164,8 @@ simplejson==3.17.2; python_version >= '2.5' and python_version not in '3.0, 3.1, six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' social-auth-core[openidconnect]==3.3.0 sortedcontainers==2.3.0 +sqlalchemy-json==0.4.0 sqlalchemy-migrate==0.13.0 -sqlalchemy-mutable==0.0.11 sqlalchemy-utils==0.36.7 sqlalchemy==1.3.20 sqlitedict==1.7.0 diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index f77c80c6274..0c87f9eee26 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -15,9 +15,11 @@ from sqlalchemy.types import ( String, TypeDecorator ) -from sqlalchemy_mutable.mutable import Mutable # For compatibility with custom yaml dumping in gxformat2 -from sqlalchemy_mutable.mutable_dict import MutableDict as MutationDict # noqa: F401 +from sqlalchemy_json import ( # noqa: F401 + mutable_json_type, + NestedMutableDict as MutationDict, +) from galaxy.util import ( smart_str, @@ -79,7 +81,7 @@ class GalaxyLargeBinary(LargeBinary): return process -class BaseJSONType(sqlalchemy.types.TypeDecorator): +class SimpleJSONType(sqlalchemy.types.TypeDecorator): """ Represents an immutable structure as a json-encoded string. @@ -115,18 +117,6 @@ class BaseJSONType(sqlalchemy.types.TypeDecorator): return (x == y) -class SimpleJSONType(BaseJSONType): - """SQLAlchemy column type that does not track mutations to mutable data.""" - pass - - -class JSONType(BaseJSONType): - """SQLAlchemy column type that tracks mutations to mutable data.""" - pass - - -Mutable.associate_with(JSONType) - metadata_pickler = AliasPickleModule({ ("cookbook.patterns", "Bunch"): ("galaxy.util.bunch", "Bunch") }) @@ -174,7 +164,7 @@ def total_size(o, handlers=None, verbose=False): return sizeof(o) -class MetadataType(JSONType): +class MetadataType(SimpleJSONType): """ Backward compatible metadata type. Can read pickles or JSON, but always writes in JSON. @@ -207,6 +197,10 @@ class MetadataType(JSONType): return ret +JSONType = mutable_json_type(dbtype=SimpleJSONType, nested=True) +MetadataType = mutable_json_type(dbtype=MetadataType, nested=True) + + class UUIDType(TypeDecorator): """ Platform-independent UUID type. diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 90ee80a21d8..280eec953b8 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -193,7 +193,7 @@ model.DynamicTool.table = Table( Column("tool_directory", Unicode(255)), Column("hidden", Boolean, default=True), Column("active", Boolean, default=True), - Column("value", JSONType()), + Column("value", JSONType), ) @@ -239,7 +239,7 @@ model.HistoryDatasetAssociation.table = Table( Column("peek", TEXT, key="_peek"), Column("tool_version", TEXT), Column("extension", TrimmedString(64)), - Column("metadata", MetadataType(), key="_metadata"), + Column("metadata", MetadataType, key="_metadata"), Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True), Column("designation", TrimmedString(255)), Column("deleted", Boolean, index=True, default=False), @@ -262,7 +262,7 @@ model.HistoryDatasetAssociationHistory.table = Table( Column("version", Integer), Column("name", TrimmedString(255)), Column("extension", TrimmedString(64)), - Column("metadata", MetadataType(), key="_metadata"), + Column("metadata", MetadataType, key="_metadata"), Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True), ) @@ -517,7 +517,7 @@ model.LibraryDatasetDatasetAssociation.table = Table( Column("peek", TEXT, key="_peek"), Column("tool_version", TEXT), Column("extension", TrimmedString(64)), - Column("metadata", MetadataType(), key="_metadata"), + Column("metadata", MetadataType, key="_metadata"), Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True), Column("designation", TrimmedString(255)), Column("deleted", Boolean, index=True, default=False), @@ -1054,7 +1054,7 @@ model.WorkflowRequestStepState.table = Table( Column("workflow_invocation_id", Integer, ForeignKey("workflow_invocation.id", onupdate="CASCADE", ondelete="CASCADE")), Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")), - Column("value", JSONType)) + Column("value", SimpleJSONType)) model.WorkflowRequestInputParameter.table = Table( "workflow_request_input_parameters", metadata, @@ -1225,9 +1225,9 @@ model.FormDefinition.table = Table( Column("name", TrimmedString(255), nullable=False), Column("desc", TEXT), Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False), - Column("fields", JSONType()), + Column("fields", JSONType), Column("type", TrimmedString(255), index=True), - Column("layout", JSONType())) + Column("layout", JSONType)) model.FormValues.table = Table( "form_values", metadata, @@ -1235,7 +1235,7 @@ model.FormValues.table = Table( Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now), Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True), - Column("content", JSONType())) + Column("content", JSONType)) model.Page.table = Table( "page", metadata, diff --git a/lib/galaxy/model/migrate/versions/0001_initial_tables.py b/lib/galaxy/model/migrate/versions/0001_initial_tables.py index 0b74124b505..6248b2ca0bb 100644 --- a/lib/galaxy/model/migrate/versions/0001_initial_tables.py +++ b/lib/galaxy/model/migrate/versions/0001_initial_tables.py @@ -58,7 +58,7 @@ HistoryDatasetAssociation_table = Table("history_dataset_association", metadata, Column("blurb", TrimmedString(255)), Column("peek", TEXT), Column("extension", TrimmedString(64)), - Column("metadata", MetadataType(), key="_metadata"), + Column("metadata", MetadataType, key="_metadata"), Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True), Column("designation", TrimmedString(255)), Column("deleted", Boolean, index=True, default=False), diff --git a/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py b/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py index 07711a97798..c79d72e94a8 100644 --- a/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py +++ b/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py @@ -170,7 +170,7 @@ LibraryDatasetDatasetAssociation_table = Table("library_dataset_dataset_associat Column("blurb", TrimmedString(255)), Column("peek", TEXT), Column("extension", TrimmedString(64)), - Column("metadata", MetadataType(), key="_metadata"), + Column("metadata", MetadataType, key="_metadata"), Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True), Column("designation", TrimmedString(255)), Column("deleted", Boolean, index=True, default=False), @@ -211,7 +211,7 @@ LibraryItemInfoTemplateElement_table = Table("library_item_info_template_element Column("description", TEXT), Column("type", TEXT, default='string'), Column("order_id", Integer), - Column("options", JSONType()), + Column("options", JSONType), Column("library_item_info_template_id", Integer, ForeignKey("library_item_info_template.id"))) Index("ix_liite_library_item_info_template_id", LibraryItemInfoTemplateElement_table.c.library_item_info_template_id) @@ -262,7 +262,7 @@ LibraryItemInfoElement_table = Table("library_item_info_element", metadata, Column("id", Integer, primary_key=True), Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now), - Column("contents", JSONType()), + Column("contents", JSONType), Column("library_item_info_id", Integer, ForeignKey("library_item_info.id"), index=True), Column("library_item_info_template_element_id", Integer, ForeignKey("library_item_info_template_element.id"))) Index("ix_liie_library_item_info_template_element_id", LibraryItemInfoElement_table.c.library_item_info_template_element_id) diff --git a/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py b/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py index 2169901a1d8..8cd1b5d40d1 100644 --- a/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py +++ b/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py @@ -44,7 +44,7 @@ FormDefinition_table = Table('form_definition', metadata, Column("name", TrimmedString(255), nullable=False), Column("desc", TEXT), Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False), - Column("fields", JSONType())) + Column("fields", JSONType)) FormDefinitionCurrent_table = Table('form_definition_current', metadata, Column("id", Integer, primary_key=True), @@ -58,7 +58,7 @@ FormValues_table = Table('form_values', metadata, Column("create_time", DateTime, default=now), Column("update_time", DateTime, default=now, onupdate=now), Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True), - Column("content", JSONType())) + Column("content", JSONType)) RequestType_table = Table('request_type', metadata, Column("id", Integer, primary_key=True), diff --git a/lib/galaxy/model/migrate/versions/0019_request_library_folder.py b/lib/galaxy/model/migrate/versions/0019_request_library_folder.py index 5471ed09f55..17cfff7b1b1 100644 --- a/lib/galaxy/model/migrate/versions/0019_request_library_folder.py +++ b/lib/galaxy/model/migrate/versions/0019_request_library_folder.py @@ -38,7 +38,7 @@ def upgrade(migrate_engine): FormDefinition_table = Table("form_definition", metadata, autoload=True) col = Column("type", TrimmedString(255), index=True) add_column(col, FormDefinition_table, metadata, index_name='ix_form_definition_type') - col = Column("layout", JSONType()) + col = Column("layout", JSONType) add_column(col, FormDefinition_table, metadata) diff --git a/lib/galaxy/model/migrate/versions/0037_samples_library.py b/lib/galaxy/model/migrate/versions/0037_samples_library.py index a51d9521bcd..154ec7c3839 100644 --- a/lib/galaxy/model/migrate/versions/0037_samples_library.py +++ b/lib/galaxy/model/migrate/versions/0037_samples_library.py @@ -35,7 +35,7 @@ def upgrade(migrate_engine): metadata.reflect() # Add the datatx_info column in 'request_type' table - col = Column("datatx_info", JSONType()) + col = Column("datatx_info", JSONType) add_column(col, 'request_type', metadata) # Delete the library_id column in 'request' table @@ -49,7 +49,7 @@ def upgrade(migrate_engine): # Add the dataset_files column in 'sample' table Sample_table = Table("sample", metadata, autoload=True) - col = Column("dataset_files", JSONType()) + col = Column("dataset_files", JSONType) add_column(col, Sample_table, metadata) # Add the library_id column in 'sample' table diff --git a/lib/galaxy/model/migrate/versions/0057_request_notify.py b/lib/galaxy/model/migrate/versions/0057_request_notify.py index ab440bca442..f28c6562132 100644 --- a/lib/galaxy/model/migrate/versions/0057_request_notify.py +++ b/lib/galaxy/model/migrate/versions/0057_request_notify.py @@ -31,7 +31,7 @@ def upgrade(migrate_engine): Request_table = Table("request", metadata, autoload=True) # create the column again as JSONType - col = Column("notification", JSONType()) + col = Column("notification", JSONType) add_column(col, Request_table, metadata) cmd = "SELECT id, user_id, notify FROM request" diff --git a/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py b/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py index cbd1280f99a..2af5e935635 100644 --- a/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py +++ b/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py @@ -221,7 +221,7 @@ def downgrade(migrate_engine): RequestType_table = Table("request_type", metadata, autoload=True) # create the 'datatx_info' column - col = Column("datatx_info", JSONType()) + col = Column("datatx_info", JSONType) add_column(col, RequestType_table, metadata) # restore the datatx_info column data in the request_type table with data from # the sequencer and the form_values table diff --git a/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py index d44d1bb6409..0d8a6cb24ac 100644 --- a/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py +++ b/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py @@ -28,7 +28,7 @@ def upgrade(migrate_engine): metadata.reflect() ToolShedRepository_table = Table("tool_shed_repository", metadata, autoload=True) - c = Column("metadata", JSONType(), nullable=True) + c = Column("metadata", JSONType, nullable=True) add_column(c, ToolShedRepository_table, metadata) c = Column("includes_datatypes", Boolean, index=True, default=False) add_column(c, ToolShedRepository_table, metadata, index_name="ix_tool_shed_repository_includes_datatypes") diff --git a/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py b/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py index 99b221d7273..4e71c5d0ba9 100644 --- a/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py +++ b/lib/galaxy/model/migrate/versions/0139_add_history_dataset_association_history_table.py @@ -35,7 +35,7 @@ HistoryDatasetAssociationHistory_table = Table( Column("version", Integer, index=True), Column("name", TrimmedString(255)), Column("extension", TrimmedString(64)), - Column("metadata", MetadataType(), key='_metadata'), + Column("metadata", MetadataType, key='_metadata'), Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True), ) diff --git a/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py b/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py index 504405dc27f..2374b2b37bf 100644 --- a/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py +++ b/lib/galaxy/model/migrate/versions/0149_dynamic_tools.py @@ -27,7 +27,7 @@ DynamicTool_table = Table( Column("tool_directory", Unicode(255)), Column("hidden", Boolean), Column("active", Boolean), - Column("value", JSONType()), + Column("value", JSONType), ) diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 1c1a0831a1d..ea9af4eede7 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -11,7 +11,7 @@ pycryptodome pysam social-auth-core[openidconnect]==3.3.0 SQLAlchemy +sqlalchemy-json sqlalchemy-migrate -sqlalchemy-mutable sqlalchemy-utils WebOb