Merge pull request #13356 from mvdbeek/release_22.01

[22.01] Merge 21.09 into 22.01

.git-blame-ignore-revs

@@ -0,0 +1,3 @@
+# Migrate code style to Prettier
+5b2928f851bd5ea3b9c2a04abf2cee9ff0bc54cc
+87873c5e2f4e6b97fe0f2084bfca0295fcd471de

lib/galaxy/jobs/__init__.py

@@ -2192,10 +2192,10 @@ class JobWrapper(HasResourceParameters):
         method should be removed ASAP and replaced with some properly generic
         and stateful way of determining link-only datasets. -nate
         """
-        if self.tool:
+        if self.tool and self.tool.id == 'upload1':
             job = self.get_job()
             param_dict = job.get_param_values(self.app)
-            return self.tool.id == 'upload1' and param_dict.get('link_data_only', None) == 'link_to_files'
+            return param_dict.get('link_data_only') == 'link_to_files'
         else:
             # The tool is unavailable, we try to move the outputs.
             return False

lib/galaxy/model/__init__.py

@@ -98,6 +98,7 @@ import galaxy.security.passwords
 import galaxy.util
 from galaxy.model.custom_types import (
     JSONType,
+    MetadataType,
     MutableJSONType,
     TrimmedString,
     UUIDType,
@@ -4402,7 +4403,7 @@ class HistoryDatasetAssociationHistory(Base, Serializable):
     version = Column(Integer)
     name = Column(TrimmedString(255))
     extension = Column(TrimmedString(64))
-    _metadata = Column('metadata', JSONType)
+    _metadata = Column("metadata", MetadataType)
     extended_metadata_id = Column(Integer, ForeignKey("extended_metadata.id"), index=True)
 
     def __init__(self,
@@ -8843,7 +8844,7 @@ HistoryDatasetAssociation.table = Table(
     Column('peek', TEXT, key='_peek'),
     Column('tool_version', TEXT),
     Column('extension', TrimmedString(64)),
-    Column('metadata', JSONType, key='_metadata'),
+    Column("metadata", MetadataType, key="_metadata"),
     Column('parent_id', Integer, ForeignKey('history_dataset_association.id'), nullable=True),
     Column('designation', TrimmedString(255)),
     Column('deleted', Boolean, index=True, default=False),
@@ -8879,7 +8880,7 @@ LibraryDatasetDatasetAssociation.table = Table(
     Column('peek', TEXT, key='_peek'),
     Column('tool_version', TEXT),
     Column('extension', TrimmedString(64)),
-    Column('metadata', JSONType, key='_metadata'),
+    Column("metadata", MetadataType, key="_metadata"),
     Column('parent_id', Integer, ForeignKey('library_dataset_dataset_association.id'), nullable=True),
     Column('designation', TrimmedString(255)),
     Column('deleted', Boolean, index=True, default=False),

lib/galaxy/model/custom_types.py

@@ -6,6 +6,7 @@ import uuid
 from collections import deque
 from itertools import chain
 from sys import getsizeof
+from typing import Optional
 
 import numpy
 import sqlalchemy
@@ -43,7 +44,7 @@ json_encoder = SafeJsonEncoder(sort_keys=True)
 json_decoder = json.JSONDecoder()
 
 # Galaxy app will set this if configured to avoid circular dependency
-MAX_METADATA_VALUE_SIZE = None
+MAX_METADATA_VALUE_SIZE: Optional[int] = None
 
 
 def _sniffnfix_pg9_hex(value):

lib/galaxy/tools/actions/__init__.py

@@ -691,12 +691,13 @@ class DefaultToolAction(ToolAction):
         return remapped_hdas
 
     def __remap_parameters(self, job_to_remap, jtid, jtod, out_data):
-        input_values = {p.name: json.loads(p.value) for p in job_to_remap.parameters}
+        input_values = {p.name: json.loads(p.value) for p in job_to_remap.parameters if p.value is not None}
         old_dataset_id = jtod.dataset_id
         new_dataset_id = out_data[jtod.name].id
         input_values = update_dataset_ids(input_values, {old_dataset_id: new_dataset_id}, src='hda')
         for p in job_to_remap.parameters:
-            p.value = json.dumps(input_values[p.name])
+            if p.name in input_values:
+                p.value = json.dumps(input_values[p.name])
         jtid.dataset = out_data[jtod.name]
         jtid.dataset.hid = jtod.dataset.hid
         log.info(f'Job {job_to_remap.id} input HDA {jtod.dataset.id} remapped to new HDA {jtid.dataset.id}')

lib/galaxy_test/api/test_workflows.py

@@ -914,6 +914,7 @@ steps:
           cond_param_inner: true
           input1:
             $link: 0/out_file1
+    thedata: null
   cat:
     tool_id: cat1
     in:

test/functional/tools/identifier_multiple_in_conditional.xml

@@ -18,6 +18,7 @@
             </when>
             <when value="false" />
         </conditional>
+        <param name="thedata" type="data" optional="true" label="Optional dummy data"/>
     </inputs>
     <outputs>
         <data name="output1" format="tabular" from_work_dir="output1" />

test/unit/data/model/mapping/test_model_mapping.py

@@ -1538,7 +1538,7 @@ class TestHistoryDatasetAssociation(BaseTest):
         peek = "e"
         tool_version = "f"
         extension = "g"
-        _metadata = "h"
+        _metadata = {"key": "value"}
         designation = "i"
         deleted = False
         visible = False
@@ -1801,7 +1801,7 @@ class TestHistoryDatasetAssociationHistory(BaseTest):
             datetime.now(),
             2,
             "b",
-            "c",
+            {"key": "value"},
         )
         obj = cls_(
             history_dataset_association.id,
@@ -3547,7 +3547,7 @@ class TestLibraryDatasetDatasetAssociation(BaseTest):
         validated_state_message = "k"
         visible = True
         message = "m"
-        _metadata = "n"
+        _metadata = {"key": "value"}
         copied_from_ldda = library_dataset_dataset_association_factory()
         parent = library_dataset_dataset_association_factory()
         persist(session, copied_from_ldda)

test/unit/data/test_metadata_limit.py

@@ -0,0 +1,44 @@
+import pytest
+
+import galaxy.datatypes.registry as registry
+import galaxy.model.mapping as mapping
+from galaxy.model import (
+    custom_types,
+    HistoryDatasetAssociation,
+    set_datatypes_registry,
+)
+
+METADATA_LIMIT = 500
+
+
+@pytest.fixture(scope="module")
+def datatypes_registry():
+    r = registry.Registry()
+    r.load_datatypes()
+    set_datatypes_registry(r)
+
+
+@pytest.fixture
+def sa_session(datatypes_registry):
+    custom_types.MAX_METADATA_VALUE_SIZE = METADATA_LIMIT
+    return mapping.init("/tmp", "sqlite:///:memory:", create_tables=True).session
+
+
+def create_bed_data(sa_session, string_size):
+    hda = HistoryDatasetAssociation(extension="bed")
+    big_string = "0" * string_size
+    sa_session.add(hda)
+    hda.metadata.column_names = [big_string]
+    assert hda.metadata.column_names
+    sa_session.flush()
+    return hda
+
+
+def test_hda_below_limit(sa_session):
+    hda = create_bed_data(sa_session=sa_session, string_size=1)
+    assert len(hda.metadata.column_names[0]) == 1
+
+
+def test_hda_above_limit(sa_session):
+    hda = create_bed_data(sa_session=sa_session, string_size=1000)
+    assert not hda.metadata.column_names