diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000000..29a799793ca --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,3 @@ +# Migrate code style to Prettier +5b2928f851bd5ea3b9c2a04abf2cee9ff0bc54cc +87873c5e2f4e6b97fe0f2084bfca0295fcd471de diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index ba4740dbf56..ec4e6adc6c9 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -2192,10 +2192,10 @@ class JobWrapper(HasResourceParameters): method should be removed ASAP and replaced with some properly generic and stateful way of determining link-only datasets. -nate """ - if self.tool: + if self.tool and self.tool.id == 'upload1': job = self.get_job() param_dict = job.get_param_values(self.app) - return self.tool.id == 'upload1' and param_dict.get('link_data_only', None) == 'link_to_files' + return param_dict.get('link_data_only') == 'link_to_files' else: # The tool is unavailable, we try to move the outputs. return False diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index c8454aa58ec..219af52d45c 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -98,6 +98,7 @@ import galaxy.security.passwords import galaxy.util from galaxy.model.custom_types import ( JSONType, + MetadataType, MutableJSONType, TrimmedString, UUIDType, @@ -4402,7 +4403,7 @@ class HistoryDatasetAssociationHistory(Base, Serializable): version = Column(Integer) name = Column(TrimmedString(255)) extension = Column(TrimmedString(64)) - _metadata = Column('metadata', JSONType) + _metadata = Column("metadata", MetadataType) extended_metadata_id = Column(Integer, ForeignKey("extended_metadata.id"), index=True) def __init__(self, @@ -8843,7 +8844,7 @@ HistoryDatasetAssociation.table = Table( Column('peek', TEXT, key='_peek'), Column('tool_version', TEXT), Column('extension', TrimmedString(64)), - Column('metadata', JSONType, key='_metadata'), + Column("metadata", MetadataType, key="_metadata"), Column('parent_id', Integer, ForeignKey('history_dataset_association.id'), nullable=True), Column('designation', TrimmedString(255)), Column('deleted', Boolean, index=True, default=False), @@ -8879,7 +8880,7 @@ LibraryDatasetDatasetAssociation.table = Table( Column('peek', TEXT, key='_peek'), Column('tool_version', TEXT), Column('extension', TrimmedString(64)), - Column('metadata', JSONType, key='_metadata'), + Column("metadata", MetadataType, key="_metadata"), Column('parent_id', Integer, ForeignKey('library_dataset_dataset_association.id'), nullable=True), Column('designation', TrimmedString(255)), Column('deleted', Boolean, index=True, default=False), diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index 1c1896ea8ac..2d762cfbf9a 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -6,6 +6,7 @@ import uuid from collections import deque from itertools import chain from sys import getsizeof +from typing import Optional import numpy import sqlalchemy @@ -43,7 +44,7 @@ json_encoder = SafeJsonEncoder(sort_keys=True) json_decoder = json.JSONDecoder() # Galaxy app will set this if configured to avoid circular dependency -MAX_METADATA_VALUE_SIZE = None +MAX_METADATA_VALUE_SIZE: Optional[int] = None def _sniffnfix_pg9_hex(value): diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 6ee80c94bca..161c5513378 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -691,12 +691,13 @@ class DefaultToolAction(ToolAction): return remapped_hdas def __remap_parameters(self, job_to_remap, jtid, jtod, out_data): - input_values = {p.name: json.loads(p.value) for p in job_to_remap.parameters} + input_values = {p.name: json.loads(p.value) for p in job_to_remap.parameters if p.value is not None} old_dataset_id = jtod.dataset_id new_dataset_id = out_data[jtod.name].id input_values = update_dataset_ids(input_values, {old_dataset_id: new_dataset_id}, src='hda') for p in job_to_remap.parameters: - p.value = json.dumps(input_values[p.name]) + if p.name in input_values: + p.value = json.dumps(input_values[p.name]) jtid.dataset = out_data[jtod.name] jtid.dataset.hid = jtod.dataset.hid log.info(f'Job {job_to_remap.id} input HDA {jtod.dataset.id} remapped to new HDA {jtid.dataset.id}') diff --git a/lib/galaxy_test/api/test_workflows.py b/lib/galaxy_test/api/test_workflows.py index 3339f55a7b6..416f7dc31b3 100644 --- a/lib/galaxy_test/api/test_workflows.py +++ b/lib/galaxy_test/api/test_workflows.py @@ -914,6 +914,7 @@ steps: cond_param_inner: true input1: $link: 0/out_file1 + thedata: null cat: tool_id: cat1 in: diff --git a/test/functional/tools/identifier_multiple_in_conditional.xml b/test/functional/tools/identifier_multiple_in_conditional.xml index c5fb19b6742..92dcfa267ba 100644 --- a/test/functional/tools/identifier_multiple_in_conditional.xml +++ b/test/functional/tools/identifier_multiple_in_conditional.xml @@ -18,6 +18,7 @@ + diff --git a/test/unit/data/model/mapping/test_model_mapping.py b/test/unit/data/model/mapping/test_model_mapping.py index 8793c46df29..bbe66d80d1c 100644 --- a/test/unit/data/model/mapping/test_model_mapping.py +++ b/test/unit/data/model/mapping/test_model_mapping.py @@ -1538,7 +1538,7 @@ class TestHistoryDatasetAssociation(BaseTest): peek = "e" tool_version = "f" extension = "g" - _metadata = "h" + _metadata = {"key": "value"} designation = "i" deleted = False visible = False @@ -1801,7 +1801,7 @@ class TestHistoryDatasetAssociationHistory(BaseTest): datetime.now(), 2, "b", - "c", + {"key": "value"}, ) obj = cls_( history_dataset_association.id, @@ -3547,7 +3547,7 @@ class TestLibraryDatasetDatasetAssociation(BaseTest): validated_state_message = "k" visible = True message = "m" - _metadata = "n" + _metadata = {"key": "value"} copied_from_ldda = library_dataset_dataset_association_factory() parent = library_dataset_dataset_association_factory() persist(session, copied_from_ldda) diff --git a/test/unit/data/test_metadata_limit.py b/test/unit/data/test_metadata_limit.py new file mode 100644 index 00000000000..e0c2121a2bc --- /dev/null +++ b/test/unit/data/test_metadata_limit.py @@ -0,0 +1,44 @@ +import pytest + +import galaxy.datatypes.registry as registry +import galaxy.model.mapping as mapping +from galaxy.model import ( + custom_types, + HistoryDatasetAssociation, + set_datatypes_registry, +) + +METADATA_LIMIT = 500 + + +@pytest.fixture(scope="module") +def datatypes_registry(): + r = registry.Registry() + r.load_datatypes() + set_datatypes_registry(r) + + +@pytest.fixture +def sa_session(datatypes_registry): + custom_types.MAX_METADATA_VALUE_SIZE = METADATA_LIMIT + return mapping.init("/tmp", "sqlite:///:memory:", create_tables=True).session + + +def create_bed_data(sa_session, string_size): + hda = HistoryDatasetAssociation(extension="bed") + big_string = "0" * string_size + sa_session.add(hda) + hda.metadata.column_names = [big_string] + assert hda.metadata.column_names + sa_session.flush() + return hda + + +def test_hda_below_limit(sa_session): + hda = create_bed_data(sa_session=sa_session, string_size=1) + assert len(hda.metadata.column_names[0]) == 1 + + +def test_hda_above_limit(sa_session): + hda = create_bed_data(sa_session=sa_session, string_size=1000) + assert not hda.metadata.column_names