mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Fix coercion error by switching to sqlalchemy-json
I was going back and forth between sqlalchemy-json and
sqlalchemy-mutable. sqlalchemy-mutable is more lenient with JSON
primitive values being set on a mutable field, so I preferred that.
However we're seeing
```
Traceback (most recent call last):
File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/__init__.py", line 1649, in handle_single_execution
rval = self.execute(
File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/__init__.py", line 1747, in execute
return self.tool_action.execute(self, trans, incoming=incoming, set_output_hid=set_output_hid, history=history, **kwargs)
File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/__init__.py", line 529, in execute
handle_output(name, output)
File "/Users/mvandenb/src/galaxy/lib/galaxy/tools/actions/__init__.py", line 442, in handle_output
data.init_meta(copy_from=metadata_source)
File "/Users/mvandenb/src/galaxy/lib/galaxy/model/__init__.py", line 2897, in init_meta
return self.datatype.init_meta(self, copy_from=copy_from)
File "/Users/mvandenb/src/galaxy/lib/galaxy/datatypes/data.py", line 181, in init_meta
dataset.metadata = copy_from.metadata
File "/Users/mvandenb/src/galaxy/lib/galaxy/model/__init__.py", line 2808, in set_metadata
self._metadata = self.metadata.make_dict_copy(bunch)
File "/Users/mvandenb/src/galaxy/.venv/lib/python3.8/site-packages/sqlalchemy/orm/attributes.py", line 272, in __set__
self.impl.set(
File "/Users/mvandenb/src/galaxy/.venv/lib/python3.8/site-packages/sqlalchemy/orm/attributes.py", line 865, in set
value = self.fire_replace_event(
File "/Users/mvandenb/src/galaxy/.venv/lib/python3.8/site-packages/sqlalchemy/orm/attributes.py", line 873, in fire_replace_event
value = fn(
File "/Users/mvandenb/src/galaxy/.venv/lib/python3.8/site-packages/sqlalchemy/ext/mutable.py", line 483, in set_
value = cls.coerce(key, value)
File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 280, in coerce
converted_obj = cls._convert(obj)
File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 317, in _convert
return tracked_type(obj, root)
File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable_dict.py", line 80, in __init__
super().__init__(self._convert_mapping(source))
File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 342, in _convert_mapping
return {
File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 343, in <dictcomp>
key: self._convert_item(item) for key, item in mapping.items()
File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 334, in _convert_item
return self._convert(item, self.root)
File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 319, in _convert
obj.root = root
File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 431, in __setattr__
return super().__setattr__(name, obj)
File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 390, in root
for child in self._tracked_children:
File "/Users/mvandenb/src/sqlalchemy-mutable/sqlalchemy_mutable/mutable.py", line 398, in _tracked_children
self.__getattribute__(name) for name in self._tracked_attr_names
AttributeError: 'MutableList' object has no attribute '_tracked_attr_names'
```
with jobs that are started on before 20.09 and that finish on 21.01.
sqlalchemy-json doesn't have that problem.
This commit is contained in:
mvdbeek
@@ -76,8 +76,8 @@ boto = "*"
|
||||
kombu = "*"
|
||||
psutil = "*"
|
||||
pulsar-galaxy-lib = "==0.14.1"
|
||||
sqlalchemy-json = "*"
|
||||
sqlalchemy-migrate = "*"
|
||||
sqlalchemy-mutable = "*"
|
||||
sqlitedict = "*"
|
||||
sqlparse = "*"
|
||||
svgwrite = "*"
|
||||
|
||||
@@ -164,8 +164,8 @@ simplejson==3.17.2; python_version >= '2.5' and python_version not in '3.0, 3.1,
|
||||
six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
social-auth-core[openidconnect]==3.3.0
|
||||
sortedcontainers==2.3.0
|
||||
sqlalchemy-json==0.4.0
|
||||
sqlalchemy-migrate==0.13.0
|
||||
sqlalchemy-mutable==0.0.11
|
||||
sqlalchemy-utils==0.36.7
|
||||
sqlalchemy==1.3.20
|
||||
sqlitedict==1.7.0
|
||||
|
||||
@@ -15,9 +15,11 @@ from sqlalchemy.types import (
|
||||
String,
|
||||
TypeDecorator
|
||||
)
|
||||
from sqlalchemy_mutable.mutable import Mutable
|
||||
# For compatibility with custom yaml dumping in gxformat2
|
||||
from sqlalchemy_mutable.mutable_dict import MutableDict as MutationDict # noqa: F401
|
||||
from sqlalchemy_json import ( # noqa: F401
|
||||
mutable_json_type,
|
||||
NestedMutableDict as MutationDict,
|
||||
)
|
||||
|
||||
from galaxy.util import (
|
||||
smart_str,
|
||||
@@ -79,7 +81,7 @@ class GalaxyLargeBinary(LargeBinary):
|
||||
return process
|
||||
|
||||
|
||||
class BaseJSONType(sqlalchemy.types.TypeDecorator):
|
||||
class SimpleJSONType(sqlalchemy.types.TypeDecorator):
|
||||
"""
|
||||
Represents an immutable structure as a json-encoded string.
|
||||
|
||||
@@ -115,18 +117,6 @@ class BaseJSONType(sqlalchemy.types.TypeDecorator):
|
||||
return (x == y)
|
||||
|
||||
|
||||
class SimpleJSONType(BaseJSONType):
|
||||
"""SQLAlchemy column type that does not track mutations to mutable data."""
|
||||
pass
|
||||
|
||||
|
||||
class JSONType(BaseJSONType):
|
||||
"""SQLAlchemy column type that tracks mutations to mutable data."""
|
||||
pass
|
||||
|
||||
|
||||
Mutable.associate_with(JSONType)
|
||||
|
||||
metadata_pickler = AliasPickleModule({
|
||||
("cookbook.patterns", "Bunch"): ("galaxy.util.bunch", "Bunch")
|
||||
})
|
||||
@@ -174,7 +164,7 @@ def total_size(o, handlers=None, verbose=False):
|
||||
return sizeof(o)
|
||||
|
||||
|
||||
class MetadataType(JSONType):
|
||||
class MetadataType(SimpleJSONType):
|
||||
"""
|
||||
Backward compatible metadata type. Can read pickles or JSON, but always
|
||||
writes in JSON.
|
||||
@@ -207,6 +197,10 @@ class MetadataType(JSONType):
|
||||
return ret
|
||||
|
||||
|
||||
JSONType = mutable_json_type(dbtype=SimpleJSONType, nested=True)
|
||||
MetadataType = mutable_json_type(dbtype=MetadataType, nested=True)
|
||||
|
||||
|
||||
class UUIDType(TypeDecorator):
|
||||
"""
|
||||
Platform-independent UUID type.
|
||||
|
||||
@@ -193,7 +193,7 @@ model.DynamicTool.table = Table(
|
||||
Column("tool_directory", Unicode(255)),
|
||||
Column("hidden", Boolean, default=True),
|
||||
Column("active", Boolean, default=True),
|
||||
Column("value", JSONType()),
|
||||
Column("value", JSONType),
|
||||
)
|
||||
|
||||
|
||||
@@ -239,7 +239,7 @@ model.HistoryDatasetAssociation.table = Table(
|
||||
Column("peek", TEXT, key="_peek"),
|
||||
Column("tool_version", TEXT),
|
||||
Column("extension", TrimmedString(64)),
|
||||
Column("metadata", MetadataType(), key="_metadata"),
|
||||
Column("metadata", MetadataType, key="_metadata"),
|
||||
Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True),
|
||||
Column("designation", TrimmedString(255)),
|
||||
Column("deleted", Boolean, index=True, default=False),
|
||||
@@ -262,7 +262,7 @@ model.HistoryDatasetAssociationHistory.table = Table(
|
||||
Column("version", Integer),
|
||||
Column("name", TrimmedString(255)),
|
||||
Column("extension", TrimmedString(64)),
|
||||
Column("metadata", MetadataType(), key="_metadata"),
|
||||
Column("metadata", MetadataType, key="_metadata"),
|
||||
Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True),
|
||||
)
|
||||
|
||||
@@ -517,7 +517,7 @@ model.LibraryDatasetDatasetAssociation.table = Table(
|
||||
Column("peek", TEXT, key="_peek"),
|
||||
Column("tool_version", TEXT),
|
||||
Column("extension", TrimmedString(64)),
|
||||
Column("metadata", MetadataType(), key="_metadata"),
|
||||
Column("metadata", MetadataType, key="_metadata"),
|
||||
Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True),
|
||||
Column("designation", TrimmedString(255)),
|
||||
Column("deleted", Boolean, index=True, default=False),
|
||||
@@ -1054,7 +1054,7 @@ model.WorkflowRequestStepState.table = Table(
|
||||
Column("workflow_invocation_id", Integer,
|
||||
ForeignKey("workflow_invocation.id", onupdate="CASCADE", ondelete="CASCADE")),
|
||||
Column("workflow_step_id", Integer, ForeignKey("workflow_step.id")),
|
||||
Column("value", JSONType))
|
||||
Column("value", SimpleJSONType))
|
||||
|
||||
model.WorkflowRequestInputParameter.table = Table(
|
||||
"workflow_request_input_parameters", metadata,
|
||||
@@ -1225,9 +1225,9 @@ model.FormDefinition.table = Table(
|
||||
Column("name", TrimmedString(255), nullable=False),
|
||||
Column("desc", TEXT),
|
||||
Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False),
|
||||
Column("fields", JSONType()),
|
||||
Column("fields", JSONType),
|
||||
Column("type", TrimmedString(255), index=True),
|
||||
Column("layout", JSONType()))
|
||||
Column("layout", JSONType))
|
||||
|
||||
model.FormValues.table = Table(
|
||||
"form_values", metadata,
|
||||
@@ -1235,7 +1235,7 @@ model.FormValues.table = Table(
|
||||
Column("create_time", DateTime, default=now),
|
||||
Column("update_time", DateTime, default=now, onupdate=now),
|
||||
Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True),
|
||||
Column("content", JSONType()))
|
||||
Column("content", JSONType))
|
||||
|
||||
model.Page.table = Table(
|
||||
"page", metadata,
|
||||
|
||||
@@ -58,7 +58,7 @@ HistoryDatasetAssociation_table = Table("history_dataset_association", metadata,
|
||||
Column("blurb", TrimmedString(255)),
|
||||
Column("peek", TEXT),
|
||||
Column("extension", TrimmedString(64)),
|
||||
Column("metadata", MetadataType(), key="_metadata"),
|
||||
Column("metadata", MetadataType, key="_metadata"),
|
||||
Column("parent_id", Integer, ForeignKey("history_dataset_association.id"), nullable=True),
|
||||
Column("designation", TrimmedString(255)),
|
||||
Column("deleted", Boolean, index=True, default=False),
|
||||
|
||||
@@ -170,7 +170,7 @@ LibraryDatasetDatasetAssociation_table = Table("library_dataset_dataset_associat
|
||||
Column("blurb", TrimmedString(255)),
|
||||
Column("peek", TEXT),
|
||||
Column("extension", TrimmedString(64)),
|
||||
Column("metadata", MetadataType(), key="_metadata"),
|
||||
Column("metadata", MetadataType, key="_metadata"),
|
||||
Column("parent_id", Integer, ForeignKey("library_dataset_dataset_association.id"), nullable=True),
|
||||
Column("designation", TrimmedString(255)),
|
||||
Column("deleted", Boolean, index=True, default=False),
|
||||
@@ -211,7 +211,7 @@ LibraryItemInfoTemplateElement_table = Table("library_item_info_template_element
|
||||
Column("description", TEXT),
|
||||
Column("type", TEXT, default='string'),
|
||||
Column("order_id", Integer),
|
||||
Column("options", JSONType()),
|
||||
Column("options", JSONType),
|
||||
Column("library_item_info_template_id", Integer, ForeignKey("library_item_info_template.id")))
|
||||
Index("ix_liite_library_item_info_template_id", LibraryItemInfoTemplateElement_table.c.library_item_info_template_id)
|
||||
|
||||
@@ -262,7 +262,7 @@ LibraryItemInfoElement_table = Table("library_item_info_element", metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
Column("create_time", DateTime, default=now),
|
||||
Column("update_time", DateTime, default=now, onupdate=now),
|
||||
Column("contents", JSONType()),
|
||||
Column("contents", JSONType),
|
||||
Column("library_item_info_id", Integer, ForeignKey("library_item_info.id"), index=True),
|
||||
Column("library_item_info_template_element_id", Integer, ForeignKey("library_item_info_template_element.id")))
|
||||
Index("ix_liie_library_item_info_template_element_id", LibraryItemInfoElement_table.c.library_item_info_template_element_id)
|
||||
|
||||
@@ -44,7 +44,7 @@ FormDefinition_table = Table('form_definition', metadata,
|
||||
Column("name", TrimmedString(255), nullable=False),
|
||||
Column("desc", TEXT),
|
||||
Column("form_definition_current_id", Integer, ForeignKey("form_definition_current.id", use_alter=True), index=True, nullable=False),
|
||||
Column("fields", JSONType()))
|
||||
Column("fields", JSONType))
|
||||
|
||||
FormDefinitionCurrent_table = Table('form_definition_current', metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
@@ -58,7 +58,7 @@ FormValues_table = Table('form_values', metadata,
|
||||
Column("create_time", DateTime, default=now),
|
||||
Column("update_time", DateTime, default=now, onupdate=now),
|
||||
Column("form_definition_id", Integer, ForeignKey("form_definition.id"), index=True),
|
||||
Column("content", JSONType()))
|
||||
Column("content", JSONType))
|
||||
|
||||
RequestType_table = Table('request_type', metadata,
|
||||
Column("id", Integer, primary_key=True),
|
||||
|
||||
@@ -38,7 +38,7 @@ def upgrade(migrate_engine):
|
||||
FormDefinition_table = Table("form_definition", metadata, autoload=True)
|
||||
col = Column("type", TrimmedString(255), index=True)
|
||||
add_column(col, FormDefinition_table, metadata, index_name='ix_form_definition_type')
|
||||
col = Column("layout", JSONType())
|
||||
col = Column("layout", JSONType)
|
||||
add_column(col, FormDefinition_table, metadata)
|
||||
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ def upgrade(migrate_engine):
|
||||
metadata.reflect()
|
||||
|
||||
# Add the datatx_info column in 'request_type' table
|
||||
col = Column("datatx_info", JSONType())
|
||||
col = Column("datatx_info", JSONType)
|
||||
add_column(col, 'request_type', metadata)
|
||||
|
||||
# Delete the library_id column in 'request' table
|
||||
@@ -49,7 +49,7 @@ def upgrade(migrate_engine):
|
||||
|
||||
# Add the dataset_files column in 'sample' table
|
||||
Sample_table = Table("sample", metadata, autoload=True)
|
||||
col = Column("dataset_files", JSONType())
|
||||
col = Column("dataset_files", JSONType)
|
||||
add_column(col, Sample_table, metadata)
|
||||
|
||||
# Add the library_id column in 'sample' table
|
||||
|
||||
@@ -31,7 +31,7 @@ def upgrade(migrate_engine):
|
||||
Request_table = Table("request", metadata, autoload=True)
|
||||
|
||||
# create the column again as JSONType
|
||||
col = Column("notification", JSONType())
|
||||
col = Column("notification", JSONType)
|
||||
add_column(col, Request_table, metadata)
|
||||
|
||||
cmd = "SELECT id, user_id, notify FROM request"
|
||||
|
||||
@@ -221,7 +221,7 @@ def downgrade(migrate_engine):
|
||||
|
||||
RequestType_table = Table("request_type", metadata, autoload=True)
|
||||
# create the 'datatx_info' column
|
||||
col = Column("datatx_info", JSONType())
|
||||
col = Column("datatx_info", JSONType)
|
||||
add_column(col, RequestType_table, metadata)
|
||||
# restore the datatx_info column data in the request_type table with data from
|
||||
# the sequencer and the form_values table
|
||||
|
||||
@@ -28,7 +28,7 @@ def upgrade(migrate_engine):
|
||||
metadata.reflect()
|
||||
|
||||
ToolShedRepository_table = Table("tool_shed_repository", metadata, autoload=True)
|
||||
c = Column("metadata", JSONType(), nullable=True)
|
||||
c = Column("metadata", JSONType, nullable=True)
|
||||
add_column(c, ToolShedRepository_table, metadata)
|
||||
c = Column("includes_datatypes", Boolean, index=True, default=False)
|
||||
add_column(c, ToolShedRepository_table, metadata, index_name="ix_tool_shed_repository_includes_datatypes")
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ HistoryDatasetAssociationHistory_table = Table(
|
||||
Column("version", Integer, index=True),
|
||||
Column("name", TrimmedString(255)),
|
||||
Column("extension", TrimmedString(64)),
|
||||
Column("metadata", MetadataType(), key='_metadata'),
|
||||
Column("metadata", MetadataType, key='_metadata'),
|
||||
Column("extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), index=True),
|
||||
)
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ DynamicTool_table = Table(
|
||||
Column("tool_directory", Unicode(255)),
|
||||
Column("hidden", Boolean),
|
||||
Column("active", Boolean),
|
||||
Column("value", JSONType()),
|
||||
Column("value", JSONType),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ pycryptodome
|
||||
pysam
|
||||
social-auth-core[openidconnect]==3.3.0
|
||||
SQLAlchemy
|
||||
sqlalchemy-json
|
||||
sqlalchemy-migrate
|
||||
sqlalchemy-mutable
|
||||
sqlalchemy-utils
|
||||
WebOb
|
||||
|
||||
Reference in New Issue
Block a user