From 49dd02f4090b525beba735fc9ff16e32388b3fcd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Jan 2018 14:10:49 +0100 Subject: [PATCH 1/6] Keep header when converting sam to bam Otherwise the converted datasets are not valid BAM files. --- lib/galaxy/datatypes/converters/sam_to_bam_native.xml | 1 + test/functional/tools/sam_to_bam_native.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/galaxy/datatypes/converters/sam_to_bam_native.xml b/lib/galaxy/datatypes/converters/sam_to_bam_native.xml index 363a3b57b68..5e8475ea52a 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam_native.xml +++ b/lib/galaxy/datatypes/converters/sam_to_bam_native.xml @@ -6,6 +6,7 @@ Date: Thu, 11 Jan 2018 14:14:15 +0100 Subject: [PATCH 2/6] Revert "fix spelling" This reverts commit b139726ed380192ecf03f10e91220a0488b573f3. --- lib/galaxy/datatypes/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 426dcb492c1..0d2b14f3808 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -801,7 +801,7 @@ class Registry(object): def get_converters_by_datatype(self, ext, priority_formats=None): """ Returns available converters by source type. - `priority_formats` will contain a list of format extensions that should + `priority_formats` will contain a lis of format extensions that should be handled with priority. This means they will end up infront of the returned ordered dictionary. """ From 38c5f711983033bb01072a769e6a618528724955 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Jan 2018 14:14:33 +0100 Subject: [PATCH 3/6] Revert "implement priority converters" This reverts commit 8928b34506c1ba1f4856fad82dc42aef0f7c68f6. --- lib/galaxy/datatypes/registry.py | 41 +++++++++++++++----------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 0d2b14f3808..72f818e55e3 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -798,34 +798,20 @@ class Registry(object): tabular.CSV() ] - def get_converters_by_datatype(self, ext, priority_formats=None): - """ - Returns available converters by source type. - `priority_formats` will contain a lis of format extensions that should - be handled with priority. This means they will end up infront of the - returned ordered dictionary. - """ + def get_converters_by_datatype(self, ext): + """Returns available converters by source type""" if ext not in self._converters_by_datatype: converters = odict() - prio_converters = odict() source_datatype = type(self.get_datatype_by_extension(ext)) for ext2, converters_dict in self.datatype_converters.items(): + converter_datatype = type(self.get_datatype_by_extension(ext2)) if issubclass(source_datatype, converter_datatype): - for k, v in converters_dict.items(): - if k in priority_formats: - prio_converters[k] = v - else: - converters[k] = v + converters.update(converters_dict) # Ensure ext-level converters are present if ext in self.datatype_converters.keys(): - for k, v in self.datatype_converters[ext].items(): - if k in priority_formats: - prio_converters[k] = v - else: - converters[k] = v - prio_converters.update(converters) - self._converters_by_datatype[ext] = prio_converters + converters.update(self.datatype_converters[ext]) + self._converters_by_datatype[ext] = converters return self._converters_by_datatype[ext] def get_converter_by_target_type(self, source_ext, target_ext): @@ -838,9 +824,20 @@ class Registry(object): def find_conversion_destination_for_dataset_by_extensions(self, dataset, accepted_formats, converter_safe=True): """Returns ( target_ext, existing converted dataset )""" - converters = self.get_converters_by_datatype(dataset.ext, [k.file_ext for k in accepted_formats]) + converters = self.get_converters_by_datatype(dataset.ext) + new_order = odict() - for convert_ext in converters: + accepted_format_keys = list() + for k in accepted_formats: + accepted_format_keys.append(k.file_ext) + if k.file_ext in converters: + new_order[k.file_ext] = converters[k.file_ext] + + for k,v in converters.items(): + if not k in accepted_format_keys: + new_order[k] = v + + for convert_ext in new_order: convert_ext_datatype = self.get_datatype_by_extension(convert_ext) if convert_ext_datatype is None: self.log.warning("Datatype class not found for extension '%s', which is used as target for conversion from datatype '%s'" % (convert_ext, dataset.ext)) From 6b444f609d3a48d17a783e581dc077b0731673ca Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 13 Jan 2018 13:10:46 +0100 Subject: [PATCH 4/6] Revert "resort the converters so that the accepted_formats are infront" This reverts commit 446643159540f8db67dfd0752560b387b63fc4b8. --- lib/galaxy/datatypes/registry.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 72f818e55e3..b893f869b73 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -804,7 +804,6 @@ class Registry(object): converters = odict() source_datatype = type(self.get_datatype_by_extension(ext)) for ext2, converters_dict in self.datatype_converters.items(): - converter_datatype = type(self.get_datatype_by_extension(ext2)) if issubclass(source_datatype, converter_datatype): converters.update(converters_dict) @@ -823,21 +822,7 @@ class Registry(object): def find_conversion_destination_for_dataset_by_extensions(self, dataset, accepted_formats, converter_safe=True): """Returns ( target_ext, existing converted dataset )""" - - converters = self.get_converters_by_datatype(dataset.ext) - new_order = odict() - - accepted_format_keys = list() - for k in accepted_formats: - accepted_format_keys.append(k.file_ext) - if k.file_ext in converters: - new_order[k.file_ext] = converters[k.file_ext] - - for k,v in converters.items(): - if not k in accepted_format_keys: - new_order[k] = v - - for convert_ext in new_order: + for convert_ext in self.get_converters_by_datatype(dataset.ext): convert_ext_datatype = self.get_datatype_by_extension(convert_ext) if convert_ext_datatype is None: self.log.warning("Datatype class not found for extension '%s', which is used as target for conversion from datatype '%s'" % (convert_ext, dataset.ext)) From c2328e32eb18bd7a4a5c791306203b933fc73603 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Jan 2018 15:04:33 +0100 Subject: [PATCH 5/6] Move up sam_to_bam_native converter That should allow implicit conversion from sam to bam_native for tools that support bam_native. --- config/datatypes_conf.xml.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 4045ac713fc..7c590f5e7df 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -288,8 +288,8 @@ - + From a36ebe35521e3e45c5c3301eef9535ebdf630f9d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 13 Jan 2018 14:17:56 +0100 Subject: [PATCH 6/6] Enhance tests for sam_to_bam conversion This demonstrates that the `sam_to_bam` converter will be used for an input with `format="bam"`, and not `sam_to_bam_native`. --- test/functional/tools/sam_to_bam_native.xml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/test/functional/tools/sam_to_bam_native.xml b/test/functional/tools/sam_to_bam_native.xml index b960bb4c87a..aed8f24892c 100644 --- a/test/functional/tools/sam_to_bam_native.xml +++ b/test/functional/tools/sam_to_bam_native.xml @@ -8,30 +8,39 @@ -b -h -@ \${GALAXY_SLOTS:-2} - -o '${output}' + -o '$bam_native_output' '$input1' - #else - cp '$input2' '$output' + #elif $input2: + cp '$input2' '$bam_native_output' + #elif $input3: + cp '$input3' '$bam_output' #end if ]]> + - + + - + - + + + + + +