diff --git a/client/src/api/datasets.ts b/client/src/api/datasets.ts index bd811255691..32826c40f49 100644 --- a/client/src/api/datasets.ts +++ b/client/src/api/datasets.ts @@ -88,6 +88,7 @@ export async function copyDataset( type, copy_elements: true, // TODO: Investigate. These should be optional, but the API requires explicit null values? + fields: null, hide_source_items: null, instance_type: null, }, diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index f9d3d4ad2e2..f3aa397fa3d 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -7067,6 +7067,12 @@ export interface components { * @description List of elements that should be in the new collection. */ element_identifiers?: components["schemas"]["CollectionElementIdentifier"][] | null; + /** + * Fields + * @description List of fields to create for this collection. Set to 'auto' to guess fields from identifiers. + * @default [] + */ + fields: string | components["schemas"]["FieldDict"][] | null; /** * Folder Id * @description The ID of the library folder that will contain the collection. Required if `instance_type=library`. @@ -7265,6 +7271,12 @@ export interface components { * @description List of elements that should be in the new collection. */ element_identifiers?: components["schemas"]["CollectionElementIdentifier"][] | null; + /** + * Fields + * @description List of fields to create for this collection. Set to 'auto' to guess fields from identifiers. + * @default [] + */ + fields: string | components["schemas"]["FieldDict"][] | null; /** * Folder Id * @description The ID of the library folder that will contain the collection. Required if `instance_type=library`. @@ -9249,6 +9261,17 @@ export interface components { /** Hash Value */ hash_value: string; }; + /** FieldDict */ + FieldDict: { + /** Format */ + format?: string | null; + /** Name */ + name: string; + /** Type */ + type: + | ("File" | "null" | "boolean" | "int" | "float" | "string") + | ("File" | "null" | "boolean" | "int" | "float" | "string")[]; + }; /** FileDataElement */ FileDataElement: { /** Md5 */ diff --git a/client/src/components/History/model/queries.ts b/client/src/components/History/model/queries.ts index 0c8a0415c3f..21ee64e4ddb 100644 --- a/client/src/components/History/model/queries.ts +++ b/client/src/components/History/model/queries.ts @@ -87,6 +87,7 @@ export async function createDatasetCollection(history: HistorySummary, inputs = copy_elements: true, name: "list", element_identifiers: [], + fields: "auto", hide_source_items: true, history_id: history.id, }; diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 18232ea3df7..5525e4de9a7 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1961,8 +1961,9 @@ class MinimalJobWrapper(HasResourceParameters): dataset.mark_unhidden() elif not purged: # If the tool was expected to set the extension, attempt to retrieve it - if dataset.ext == "auto": - dataset.extension = context.get("ext", "data") + context_ext = context.get("ext", "data") + if dataset.ext == "auto" or (dataset.ext == "data" and context_ext != "data"): + dataset.extension = context_ext dataset.init_meta(copy_from=dataset) # if a dataset was copied, it won't appear in our dictionary: # either use the metadata from originating output dataset, or call set_meta on the copies diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py index 04825abc66a..d711f558a1d 100644 --- a/lib/galaxy/managers/collections.py +++ b/lib/galaxy/managers/collections.py @@ -178,6 +178,7 @@ class DatasetCollectionManager: flush=True, completed_job=None, output_name=None, + fields=None, ): """ PRECONDITION: security checks on ability to add to parent @@ -202,6 +203,7 @@ class DatasetCollectionManager: hide_source_items=hide_source_items, copy_elements=copy_elements, history=history, + fields=fields, ) implicit_inputs = [] @@ -288,17 +290,20 @@ class DatasetCollectionManager: hide_source_items=None, copy_elements=False, history=None, + fields=None, ): # Make sure at least one of these is None. assert element_identifiers is None or elements is None - if element_identifiers is None and elements is None: raise RequestParameterInvalidException(ERROR_INVALID_ELEMENTS_SPECIFICATION) if not collection_type: raise RequestParameterInvalidException(ERROR_NO_COLLECTION_TYPE) - collection_type_description = self.collection_type_descriptions.for_collection_type(collection_type) + collection_type_description = self.collection_type_descriptions.for_collection_type( + collection_type, fields=fields + ) has_subcollections = collection_type_description.has_subcollections() + # If we have elements, this is an internal request, don't need to load # objects from identifiers. if elements is None: @@ -322,8 +327,9 @@ class DatasetCollectionManager: if elements is not self.ELEMENTS_UNINITIALIZED: type_plugin = collection_type_description.rank_type_plugin() - dataset_collection = builder.build_collection(type_plugin, elements) + dataset_collection = builder.build_collection(type_plugin, elements, fields=fields) else: + # TODO: Pass fields here - need test case first. dataset_collection = model.DatasetCollection(populated=False) dataset_collection.collection_type = collection_type return dataset_collection diff --git a/lib/galaxy/managers/collections_util.py b/lib/galaxy/managers/collections_util.py index 32feab23889..7f129992c75 100644 --- a/lib/galaxy/managers/collections_util.py +++ b/lib/galaxy/managers/collections_util.py @@ -39,6 +39,7 @@ def api_payload_to_create_params(payload): name=payload.get("name", None), hide_source_items=string_as_bool(payload.get("hide_source_items", False)), copy_elements=string_as_bool(payload.get("copy_elements", False)), + fields=payload.get("fields", None), ) return params diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index d3f6177faf9..20c9e77e5dd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -6614,6 +6614,7 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): element_count: Mapped[Optional[int]] create_time: Mapped[datetime] = mapped_column(default=now, nullable=True) update_time: Mapped[datetime] = mapped_column(default=now, onupdate=now, nullable=True) + fields: Mapped[Optional[bytes]] = mapped_column(JSONType, nullable=True) elements: Mapped[List["DatasetCollectionElement"]] = relationship( primaryjoin=(lambda: DatasetCollection.id == DatasetCollectionElement.dataset_collection_id), @@ -6626,12 +6627,21 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): populated_states = DatasetCollectionPopulatedState - def __init__(self, id=None, collection_type=None, populated=True, element_count=None): + def __init__( + self, + id=None, + collection_type=None, + populated=True, + element_count=None, + fields=None, + ): self.id = id self.collection_type = collection_type if not populated: self.populated_state = DatasetCollection.populated_states.NEW self.element_count = element_count + # TODO: persist fields... + self.fields = fields def _build_nested_collection_attributes_stmt( self, @@ -6806,6 +6816,10 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): return self._populated_optimized + @property + def allow_implicit_mapping(self): + return self.collection_type != "record" + @property def populated(self): top_level_populated = self.populated_state == DatasetCollection.populated_states.OK diff --git a/lib/galaxy/model/dataset_collections/builder.py b/lib/galaxy/model/dataset_collections/builder.py index 2ae001f33a2..8c7f2b1e500 100644 --- a/lib/galaxy/model/dataset_collections/builder.py +++ b/lib/galaxy/model/dataset_collections/builder.py @@ -1,28 +1,37 @@ +from typing import ( + Any, + Dict, + List, +) + from galaxy import model from galaxy.model.orm.util import add_object_to_object_session +from galaxy.tool_util_models.tool_source import FieldDict from galaxy.util.oset import OrderedSet from .type_description import COLLECTION_TYPE_DESCRIPTION_FACTORY -def build_collection(type, dataset_instances, collection=None, associated_identifiers=None): +def build_collection(type, dataset_instances, collection=None, associated_identifiers=None, fields=None): """ Build DatasetCollection with populated DatasetcollectionElement objects corresponding to the supplied dataset instances or throw exception if this is not a valid collection of the specified type. """ - dataset_collection = collection or model.DatasetCollection() + dataset_collection = collection or model.DatasetCollection(fields=fields) associated_identifiers = associated_identifiers or set() - set_collection_elements(dataset_collection, type, dataset_instances, associated_identifiers) + set_collection_elements(dataset_collection, type, dataset_instances, associated_identifiers, fields=fields) return dataset_collection -def set_collection_elements(dataset_collection, type, dataset_instances, associated_identifiers): +def set_collection_elements(dataset_collection, type, dataset_instances, associated_identifiers, fields=None): new_element_keys = OrderedSet(dataset_instances.keys()) - associated_identifiers new_dataset_instances = {k: dataset_instances[k] for k in new_element_keys} dataset_collection.element_count = dataset_collection.element_count or 0 element_index = dataset_collection.element_count elements = [] - for element in type.generate_elements(new_dataset_instances): + if type.collection_type == "record" and fields == "auto": + fields = guess_fields(dataset_instances) + for element in type.generate_elements(new_dataset_instances, fields=fields): element.element_index = element_index add_object_to_object_session(element, dataset_collection) element.collection = dataset_collection @@ -35,6 +44,16 @@ def set_collection_elements(dataset_collection, type, dataset_instances, associa return dataset_collection +def guess_fields(dataset_instances: Dict[str, Any]) -> List[FieldDict]: + fields: List[FieldDict] = [] + for identifier, element in dataset_instances.items(): + # TODO: Make generic enough to handle nested record types. + assert element.history_content_type == "dataset" + fields.append({"type": "File", "name": identifier}) + + return fields + + class CollectionBuilder: """Purely functional builder pattern for building a dataset collection.""" diff --git a/lib/galaxy/model/dataset_collections/registry.py b/lib/galaxy/model/dataset_collections/registry.py index 9c849dfdad6..bd148edafd2 100644 --- a/lib/galaxy/model/dataset_collections/registry.py +++ b/lib/galaxy/model/dataset_collections/registry.py @@ -2,9 +2,14 @@ from galaxy import model from .types import ( list, paired, + record, ) -PLUGIN_CLASSES = [list.ListDatasetCollectionType, paired.PairedDatasetCollectionType] +PLUGIN_CLASSES = [ + list.ListDatasetCollectionType, + paired.PairedDatasetCollectionType, + record.RecordDatasetCollectionType, +] class DatasetCollectionTypesRegistry: @@ -14,13 +19,13 @@ class DatasetCollectionTypesRegistry: def get(self, plugin_type): return self.__plugins[plugin_type] - def prototype(self, plugin_type): + def prototype(self, plugin_type, fields=None): plugin_type_object = self.get(plugin_type) if not hasattr(plugin_type_object, "prototype_elements"): raise Exception(f"Cannot pre-determine structure for collection of type {plugin_type}") dataset_collection = model.DatasetCollection() - for e in plugin_type_object.prototype_elements(): + for e in plugin_type_object.prototype_elements(fields=fields): e.collection = dataset_collection return dataset_collection diff --git a/lib/galaxy/model/dataset_collections/type_description.py b/lib/galaxy/model/dataset_collections/type_description.py index cade102453c..172ac1976b2 100644 --- a/lib/galaxy/model/dataset_collections/type_description.py +++ b/lib/galaxy/model/dataset_collections/type_description.py @@ -9,9 +9,9 @@ class CollectionTypeDescriptionFactory: # I think. self.type_registry = type_registry - def for_collection_type(self, collection_type): + def for_collection_type(self, collection_type, fields=None): assert collection_type is not None - return CollectionTypeDescription(collection_type, self) + return CollectionTypeDescription(collection_type, self, fields=fields) class CollectionTypeDescription: @@ -47,12 +47,15 @@ class CollectionTypeDescription: collection_type: str - def __init__(self, collection_type: Union[str, "CollectionTypeDescription"], collection_type_description_factory): + def __init__( + self, collection_type: Union[str, "CollectionTypeDescription"], collection_type_description_factory, fields=None + ): if isinstance(collection_type, CollectionTypeDescription): self.collection_type = collection_type.collection_type else: self.collection_type = collection_type self.collection_type_description_factory = collection_type_description_factory + self.fields = fields self.__has_subcollections = self.collection_type.find(":") > 0 def child_collection_type(self): diff --git a/lib/galaxy/model/dataset_collections/types/__init__.py b/lib/galaxy/model/dataset_collections/types/__init__.py index bfcf7bae79a..c294f6957be 100644 --- a/lib/galaxy/model/dataset_collections/types/__init__.py +++ b/lib/galaxy/model/dataset_collections/types/__init__.py @@ -11,7 +11,7 @@ log = logging.getLogger(__name__) class DatasetCollectionType(metaclass=ABCMeta): @abstractmethod - def generate_elements(self, dataset_instances): + def generate_elements(self, dataset_instances: dict, **kwds): """Generate DatasetCollectionElements with corresponding to the supplied dataset instances or throw exception if this is not a valid collection of the specified type. diff --git a/lib/galaxy/model/dataset_collections/types/list.py b/lib/galaxy/model/dataset_collections/types/list.py index 18ce4db7653..d4421d009c3 100644 --- a/lib/galaxy/model/dataset_collections/types/list.py +++ b/lib/galaxy/model/dataset_collections/types/list.py @@ -7,8 +7,8 @@ class ListDatasetCollectionType(BaseDatasetCollectionType): collection_type = "list" - def generate_elements(self, elements): - for identifier, element in elements.items(): + def generate_elements(self, dataset_instances, **kwds): + for identifier, element in dataset_instances.items(): association = DatasetCollectionElement( element=element, element_identifier=identifier, diff --git a/lib/galaxy/model/dataset_collections/types/paired.py b/lib/galaxy/model/dataset_collections/types/paired.py index 4ae95a1442a..e774ab67aac 100644 --- a/lib/galaxy/model/dataset_collections/types/paired.py +++ b/lib/galaxy/model/dataset_collections/types/paired.py @@ -15,21 +15,21 @@ class PairedDatasetCollectionType(BaseDatasetCollectionType): collection_type = "paired" - def generate_elements(self, elements): - if forward_dataset := elements.get(FORWARD_IDENTIFIER): + def generate_elements(self, dataset_instances, **kwds): + if forward_dataset := dataset_instances.get(FORWARD_IDENTIFIER): left_association = DatasetCollectionElement( element=forward_dataset, element_identifier=FORWARD_IDENTIFIER, ) yield left_association - if reverse_dataset := elements.get(REVERSE_IDENTIFIER): + if reverse_dataset := dataset_instances.get(REVERSE_IDENTIFIER): right_association = DatasetCollectionElement( element=reverse_dataset, element_identifier=REVERSE_IDENTIFIER, ) yield right_association - def prototype_elements(self): + def prototype_elements(self, **kwds): left_association = DatasetCollectionElement( element=HistoryDatasetAssociation(), element_identifier=FORWARD_IDENTIFIER, diff --git a/lib/galaxy/model/dataset_collections/types/record.py b/lib/galaxy/model/dataset_collections/types/record.py new file mode 100644 index 00000000000..193509f439e --- /dev/null +++ b/lib/galaxy/model/dataset_collections/types/record.py @@ -0,0 +1,45 @@ +from galaxy.exceptions import RequestParameterMissingException +from galaxy.model import ( + DatasetCollectionElement, + HistoryDatasetAssociation, +) +from ..types import BaseDatasetCollectionType + + +class RecordDatasetCollectionType(BaseDatasetCollectionType): + """Arbitrary CWL-style record type.""" + + collection_type = "record" + + def generate_elements(self, dataset_instances, **kwds): + fields = kwds.get("fields", None) + if fields is None: + raise RequestParameterMissingException("Missing or null parameter 'fields' required for record types.") + if len(dataset_instances) != len(fields): + self._validation_failed("Supplied element do not match fields.") + index = 0 + for identifier, element in dataset_instances.items(): + field = fields[index] + if field["name"] != identifier: + self._validation_failed("Supplied element do not match fields.") + + # TODO: validate type and such. + association = DatasetCollectionElement( + element=element, + element_identifier=identifier, + ) + yield association + index += 1 + + def prototype_elements(self, fields=None, **kwds): + if fields is None: + raise RequestParameterMissingException("Missing or null parameter 'fields' required for record types.") + for field in fields: + name = field.get("name", None) + assert name + assert field.get("type", "File") # NS: this assert doesn't make sense as it is + field_dataset = DatasetCollectionElement( + element=HistoryDatasetAssociation(), + element_identifier=name, + ) + yield field_dataset diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index fcf28ebe061..6ab318d11b3 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -50,6 +50,7 @@ from galaxy.schema.types import ( OffsetNaiveDatetime, RelativeUrl, ) +from galaxy.tool_util_models.tool_source import FieldDict from galaxy.util.hash_util import HashFunctionNameEnum from galaxy.util.sanitize_html import sanitize_html @@ -1715,6 +1716,11 @@ class CreateNewCollectionPayload(Model): default=None, description="The ID of the library folder that will contain the collection. Required if `instance_type=library`.", ) + fields_: Optional[Union[str, List[FieldDict]]] = Field( + default=[], + description="List of fields to create for this collection. Set to 'auto' to guess fields from identifiers.", + alias="fields", + ) class ModelStoreFormat(str, Enum): diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py index 35087f97f5f..b0594f4acc0 100644 --- a/lib/galaxy/tool_util/parser/interface.py +++ b/lib/galaxy/tool_util/parser/interface.py @@ -31,6 +31,7 @@ from galaxy.tool_util_models.tool_source import ( BaseJsonTestCollectionDefCollectionElementDict, Citation, DrillDownOptionsDict, + FieldDict, HelpContent, JsonTestCollectionDefCollectionElementDict, JsonTestCollectionDefDatasetElementDict, @@ -597,6 +598,7 @@ class XmlTestCollectionDefDict(TypedDict): model_class: Literal["TestCollectionDef"] attributes: TestCollectionAttributeDict collection_type: CollectionType + fields: Optional[List[FieldDict]] elements: List[TestCollectionDefElementDict] name: str @@ -634,12 +636,17 @@ def _copy_if_exists(attributes, as_dict, name: str, as_name: Optional[str] = Non class TestCollectionDef: __test__ = False # Prevent pytest from discovering this class (issue #12071) elements: List[TestCollectionDefElementInternal] + collection_type: Optional[str] + fields: Optional[List[FieldDict]] - def __init__(self, attrib, name, collection_type, elements): + def __init__( + self, attrib, name, collection_type: Optional[str], elements, fields: Optional[List[FieldDict]] = None + ): self.attrib = attrib self.collection_type = collection_type self.elements = elements self.name = name + self.fields = fields def _test_format_to_dict(self) -> "BaseJsonTestCollectionDefCollectionElementDict": @@ -696,6 +703,7 @@ class TestCollectionDef: "collection_type": self.collection_type, "elements": list(map(element_to_dict, self.elements or [])), "name": self.name, + "fields": self.fields, } @staticmethod @@ -719,6 +727,7 @@ class TestCollectionDef: name=xml_as_dict.get("name", "Unnamed Collection"), elements=list(map(element_from_dict, xml_as_dict["elements"] or [])), collection_type=xml_as_dict["collection_type"], + fields=xml_as_dict.get("fields", None), ) else: json_as_dict = cast(JsonTestCollectionDefDict, as_dict) diff --git a/lib/galaxy/tool_util/parser/output_objects.py b/lib/galaxy/tool_util/parser/output_objects.py index 090b38ac63b..655c0051854 100644 --- a/lib/galaxy/tool_util/parser/output_objects.py +++ b/lib/galaxy/tool_util/parser/output_objects.py @@ -425,12 +425,14 @@ class ToolOutputCollectionStructure: collection_type_from_rules: Optional[str] = None, structured_like: Optional[str] = None, dataset_collector_descriptions: Optional[List[DatasetCollectionDescription]] = None, + fields=None, ) -> None: self.collection_type = collection_type self.collection_type_source = collection_type_source self.collection_type_from_rules = collection_type_from_rules self.structured_like = structured_like self.dataset_collector_descriptions = dataset_collector_descriptions or [] + self.fields = fields if collection_type and collection_type_source: raise ValueError("Cannot set both type and type_source on collection output.") if ( @@ -447,6 +449,10 @@ class ToolOutputCollectionStructure: raise ValueError( "Cannot specify dynamic structure (discover_datasets) and collection type attributes structured_like or collection_type_from_rules." ) + if collection_type == "record" and fields is None: + raise ValueError("If record outputs are defined, fields must be defined as well.") + if fields is not None and collection_type != "record": + raise ValueError("If fields are specified for outputs, the collection type must be record.") self.dynamic = bool(dataset_collector_descriptions) def collection_prototype(self, inputs, type_registry): @@ -456,7 +462,7 @@ class ToolOutputCollectionStructure: else: collection_type = self.collection_type assert collection_type - collection_prototype = type_registry.prototype(collection_type) + collection_prototype = type_registry.prototype(collection_type, fields=self.fields) collection_prototype.collection_type = collection_type return collection_prototype diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 1b942b3070a..1d753fb4db3 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -1045,13 +1045,18 @@ def _test_collection_def_dict(elem: Element) -> XmlTestCollectionDefDict: else: element_definition = __parse_param_elem(element) elements.append({"element_identifier": element_identifier, "element_definition": element_definition}) - + fields_json = "null" + fields_el = elem.find("fields") + if fields_el is not None: + fields_json = fields_el.text or "null" + fields = json.loads(fields_json) return XmlTestCollectionDefDict( model_class="TestCollectionDef", attributes=attrib, collection_type=collection_type, elements=elements, name=name, + fields=fields, ) diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index 9d9f87a6f4a..650e5c80682 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -693,6 +693,8 @@ class GalaxyInteractorApi: collection_type=collection_def.collection_type, history_id=history_id, ) + if collection_def.fields: + create_payload["fields"] = collection_def.fields return self._post("dataset_collections", data=create_payload, json=True).json()["id"] def _element_identifiers(self, collection_def): diff --git a/lib/galaxy/tool_util_models/tool_source.py b/lib/galaxy/tool_util_models/tool_source.py index d7bb207b58d..c7f76d86326 100644 --- a/lib/galaxy/tool_util_models/tool_source.py +++ b/lib/galaxy/tool_util_models/tool_source.py @@ -44,6 +44,18 @@ class DrillDownOptionsDict(TypedDict): selected: bool +# For fields... just implementing a subset of CWL for Galaxy flavors of these objects +# so far. +CwlType = Literal["File", "null", "boolean", "int", "float", "string"] +FieldType = Union[CwlType, List[CwlType]] + + +class FieldDict(TypedDict): + name: str + type: FieldType + format: NotRequired[Optional[str]] + + JsonTestDatasetDefDict = TypedDict( "JsonTestDatasetDefDict", { @@ -81,7 +93,7 @@ BaseJsonTestCollectionDefCollectionElementDict = TypedDict( "BaseJsonTestCollectionDefCollectionElementDict", { "class": Literal["Collection"], - "collection_type": str, + "collection_type": Optional[str], "elements": NotRequired[Optional[List[JsonTestCollectionDefElementDict]]], }, ) @@ -91,7 +103,7 @@ JsonTestCollectionDefCollectionElementDict = TypedDict( { "identifier": str, "class": Literal["Collection"], - "collection_type": str, + "collection_type": Optional[str], "elements": NotRequired[Optional[List[JsonTestCollectionDefElementDict]]], }, ) @@ -100,7 +112,7 @@ JsonTestCollectionDefDict = TypedDict( "JsonTestCollectionDefDict", { "class": Literal["Collection"], - "collection_type": str, + "collection_type": Optional[str], "elements": NotRequired[Optional[List[JsonTestCollectionDefElementDict]]], "name": NotRequired[Optional[str]], }, diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index a97fdb4d4af..e3706e0c4f9 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3760,7 +3760,7 @@ class ExtractDatasetCollectionTool(DatabaseOperationTool): collection = has_collection.collection collection_type = collection.collection_type - assert collection_type in ["list", "paired"] + assert collection_type in ["list", "paired", "record"] how = incoming["which"]["which_dataset"] if how == "first": extracted_element = collection.first_dataset_element diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index d434df8b287..1ad5b688411 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -686,7 +686,10 @@ class DefaultToolAction(ToolAction): assert not element_identifiers # known_outputs must have been empty element_kwds = dict(elements=collections_manager.ELEMENTS_UNINITIALIZED) else: - element_kwds = dict(element_identifiers=element_identifiers) + element_kwds = dict( + element_identifiers=element_identifiers, + fields=output.structure.fields, + ) output_collections.create_collection( output=output, name=name, completed_job=completed_job, **element_kwds ) diff --git a/lib/galaxy/tools/extract_dataset.xml b/lib/galaxy/tools/extract_dataset.xml index 37986eb722f..5c7ece160ab 100644 --- a/lib/galaxy/tools/extract_dataset.xml +++ b/lib/galaxy/tools/extract_dataset.xml @@ -10,7 +10,7 @@ operation_3695 - + @@ -73,6 +73,18 @@ This tool will create new history datasets from your collection but your quota u + + + + [{"name": "parent", "type": "File"}, {"name": "child", "type": "File"}] + + + + + + + + diff --git a/lib/galaxy/webapps/galaxy/services/dataset_collections.py b/lib/galaxy/webapps/galaxy/services/dataset_collections.py index 273012052ad..c237d04f84f 100644 --- a/lib/galaxy/webapps/galaxy/services/dataset_collections.py +++ b/lib/galaxy/webapps/galaxy/services/dataset_collections.py @@ -123,7 +123,7 @@ class DatasetCollectionsService(ServiceBase, UsesLibraryMixinItems): :returns: element view of new dataset collection """ # TODO: Error handling... - create_params = api_payload_to_create_params(payload.dict(exclude_unset=True)) + create_params = api_payload_to_create_params(payload.dict(exclude_unset=True, by_alias=True)) if payload.instance_type == "history": if payload.history_id is None: raise exceptions.RequestParameterInvalidException("Parameter history_id is required.") diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index f7b0c188c64..4a9d1d2cc5b 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -565,7 +565,7 @@ class WorkflowModule: for input_dict in all_inputs: name = input_dict["name"] data = progress.replacement_for_input(self.trans, step, input_dict) - can_map_over = hasattr(data, "collection") # and data.collection.allow_implicit_mapping + can_map_over = hasattr(data, "collection") and data.collection.allow_implicit_mapping if not can_map_over: continue diff --git a/lib/galaxy_test/api/test_dataset_collections.py b/lib/galaxy_test/api/test_dataset_collections.py index d7710c57b2f..372d693d4fd 100644 --- a/lib/galaxy_test/api/test_dataset_collections.py +++ b/lib/galaxy_test/api/test_dataset_collections.py @@ -1,3 +1,4 @@ +import json import zipfile from io import BytesIO from typing import List @@ -100,6 +101,106 @@ class TestDatasetCollectionsApi(ApiTestCase): pair_1_element_1 = pair_elements[0] assert pair_1_element_1["element_index"] == 0 + def test_create_record(self, history_id): + contents = [ + ("condition", "1\t2\t3"), + ("control1", "4\t5\t6"), + ("control2", "7\t8\t9"), + ] + record_identifiers = self.dataset_collection_populator.list_identifiers(history_id, contents) + fields = [ + {"name": "condition", "type": "File"}, + {"name": "control1", "type": "File"}, + {"name": "control2", "type": "File"}, + ] + payload = dict( + name="a record", + instance_type="history", + history_id=history_id, + element_identifiers=record_identifiers, + collection_type="record", + fields=fields, + ) + create_response = self._post("dataset_collections", payload, json=True) + dataset_collection = self._check_create_response(create_response) + assert dataset_collection["collection_type"] == "record" + assert dataset_collection["name"] == "a record" + returned_collections = dataset_collection["elements"] + assert len(returned_collections) == 3, dataset_collection + record_pos_0_element = returned_collections[0] + self._assert_has_keys(record_pos_0_element, "element_index") + record_pos_0_object = record_pos_0_element["object"] + self._assert_has_keys(record_pos_0_object, "name", "history_content_type") + + def test_record_requires_fields(self, history_id): + contents = [ + ("condition", "1\t2\t3"), + ("control1", "4\t5\t6"), + ("control2", "7\t8\t9"), + ] + record_identifiers = self.dataset_collection_populator.list_identifiers(history_id, contents) + payload = dict( + name="a record", + instance_type="history", + history_id=history_id, + element_identifiers=json.dumps(record_identifiers), + collection_type="record", + ) + create_response = self._post("dataset_collections", payload) + self._assert_status_code_is(create_response, 400) + + def test_record_auto_fields(self, history_id): + contents = [ + ("condition", "1\t2\t3"), + ("control1", "4\t5\t6"), + ("control2", "7\t8\t9"), + ] + record_identifiers = self.dataset_collection_populator.list_identifiers(history_id, contents) + payload = dict( + name="a record", + instance_type="history", + history_id=history_id, + element_identifiers=record_identifiers, + collection_type="record", + fields="auto", + ) + create_response = self._post("dataset_collections", payload, json=True) + self._check_create_response(create_response) + + def test_record_field_validation(self, history_id): + contents = [ + ("condition", "1\t2\t3"), + ("control1", "4\t5\t6"), + ("control2", "7\t8\t9"), + ] + record_identifiers = self.dataset_collection_populator.list_identifiers(history_id, contents) + too_few_fields = [ + {"name": "condition", "type": "File"}, + {"name": "control1", "type": "File"}, + ] + too_many_fields = [ + {"name": "condition", "type": "File"}, + {"name": "control1", "type": "File"}, + {"name": "control2", "type": "File"}, + {"name": "control3", "type": "File"}, + ] + wrong_name_fields = [ + {"name": "condition", "type": "File"}, + {"name": "control1", "type": "File"}, + {"name": "control3", "type": "File"}, + ] + for fields in [too_few_fields, too_many_fields, wrong_name_fields]: + payload = dict( + name="a record", + instance_type="history", + history_id=history_id, + element_identifiers=json.dumps(record_identifiers), + collection_type="record", + fields=json.dumps(fields), + ) + create_response = self._post("dataset_collections", payload) + self._assert_status_code_is(create_response, 400) + def test_list_download(self): with self.dataset_populator.test_history(require_new=False) as history_id: fetch_response = self.dataset_collection_populator.create_list_in_history( diff --git a/test/functional/tools/collection_record_test_two_files.xml b/test/functional/tools/collection_record_test_two_files.xml new file mode 100644 index 00000000000..ae69549fe00 --- /dev/null +++ b/test/functional/tools/collection_record_test_two_files.xml @@ -0,0 +1,32 @@ + + + cat $f1.parent $f1['child'] >> $out1; + echo 'Collection name: $f1.name' + + + + + + + + + + + + [{"name": "parent", "type": "File"}, {"name": "child", "type": "File"}] + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/sample_tool_conf.xml b/test/functional/tools/sample_tool_conf.xml index 20aa9225491..f557120cb62 100644 --- a/test/functional/tools/sample_tool_conf.xml +++ b/test/functional/tools/sample_tool_conf.xml @@ -215,6 +215,7 @@ + diff --git a/test/unit/tool_util/test_test_definition_parsing.py b/test/unit/tool_util/test_test_definition_parsing.py index ed5fbdaaee9..77cda95c737 100644 --- a/test/unit/tool_util/test_test_definition_parsing.py +++ b/test/unit/tool_util/test_test_definition_parsing.py @@ -58,6 +58,13 @@ COLLECTION_TYPE_SOURCE_EXPECTATIONS = [ BIGWIG_TO_WIG_EXPECTATIONS = [ (["inputs", "chrom"], "chr21"), ] +RECORD_TWO_FILES_EXPECTATIONS = [ + (["inputs", "f1", "model_class"], "TestCollectionDef"), + (["inputs", "f1", "collection_type"], "record"), + (["inputs", "f1", "fields", 0, "type"], "File"), + (["inputs", "f1", "fields", 0, "name"], "parent"), + (["inputs", "f1", "fields", 1, "type"], "File"), +] class TestTestParsing(TestCase): @@ -118,6 +125,12 @@ class TestTestParsing(TestCase): test_0 = test_dicts[0].to_dict() assert test_0["error"] is True + def test_field_collection_inputs(self): + self._init_tool_for_path(functional_test_tool_path("collection_record_test_two_files.xml")) + test_dicts = self._parse_tests() + test_0 = test_dicts[0].to_dict() + self._verify_each(test_0, RECORD_TWO_FILES_EXPECTATIONS) + def test_bigwigtowig_converter(self): # defines if in_packages():