diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 638d8cd66a8..5aa21dcf169 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -477,9 +477,9 @@ class Bed(Interval): if not get_headers(file_prefix, '\t', comment_designator='#', count=1): return False try: - headers = iter_headers(file_prefix, '\t', comment_designator='#') - for hdr in headers: - if hdr[0] == '': + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t', comment_designator='#'): + if not hdr or hdr == ['']: continue if len(hdr) < 3 or len(hdr) > 12: return False @@ -542,7 +542,8 @@ class Bed(Interval): return False if len(block_sizes) != block_count or len(block_starts) != block_count: return False - return True + found_valid_lines = True + return found_valid_lines except Exception: return False @@ -818,28 +819,33 @@ class Gff(Tabular, _RemoteCallMixin): if len(get_headers(file_prefix, '\t', count=2)) < 2: return False try: - headers = iter_headers(file_prefix, '\t') - for hdr in headers: - if hdr and hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t'): + if not hdr or hdr == ['']: + continue + if hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: return False - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != 9: - return False + # The gff-version header comment may have been stripped, so inspect the data + if hdr[0].startswith('#'): + continue + if len(hdr) != 9: + return False + try: + int(hdr[3]) + int(hdr[4]) + except Exception: + return False + if hdr[5] != '.': try: - int(hdr[3]) - int(hdr[4]) + float(hdr[5]) except Exception: return False - if hdr[5] != '.': - try: - float(hdr[5]) - except Exception: - return False - if hdr[6] not in data.valid_strand: - return False - if hdr[7] not in self.valid_gff_frame: - return False - return True + if hdr[6] not in data.valid_strand: + return False + if hdr[7] not in self.valid_gff_frame: + return False + found_valid_lines = True + return found_valid_lines except Exception: return False @@ -953,37 +959,41 @@ class Gff3(Gff): if len(get_headers(file_prefix, '\t', count=2)) < 2: return False try: - headers = iter_headers(file_prefix, '\t') - for hdr in headers: - if hdr and hdr[0].startswith('##gff-version') and hdr[0].find('3') >= 0: + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t'): + if not hdr or hdr == ['']: + continue + if hdr[0].startswith('##gff-version') and hdr[0].find('3') >= 0: return True - elif hdr and hdr[0].startswith('##gff-version') and hdr[0].find('3') < 0: + elif hdr[0].startswith('##gff-version') and hdr[0].find('3') < 0: return False - # Header comments may have been stripped, so inspect the data - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != 9: + # The gff-version header comment may have been stripped, so inspect the data + if hdr[0].startswith('#'): + continue + if len(hdr) != 9: + return False + try: + int(hdr[3]) + except Exception: + if hdr[3] != '.': return False + try: + int(hdr[4]) + except Exception: + if hdr[4] != '.': + return False + if hdr[5] != '.': try: - int(hdr[3]) + float(hdr[5]) except Exception: - if hdr[3] != '.': - return False - try: - int(hdr[4]) - except Exception: - if hdr[4] != '.': - return False - if hdr[5] != '.': - try: - float(hdr[5]) - except Exception: - return False - if hdr[6] not in self.valid_gff3_strand: return False - if hdr[7] not in self.valid_gff3_phase: - return False - parse_gff3_attributes(hdr[8]) - return True + if hdr[6] not in self.valid_gff3_strand: + return False + if hdr[7] not in self.valid_gff3_phase: + return False + parse_gff3_attributes(hdr[8]) + found_valid_lines = True + return found_valid_lines except Exception: return False @@ -1031,34 +1041,38 @@ class Gtf(Gff): if len(get_headers(file_prefix, '\t', count=2)) < 2: return False try: - headers = iter_headers(file_prefix, '\t') - for hdr in headers: - if hdr and hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: + found_valid_lines = False + for hdr in iter_headers(file_prefix, '\t'): + if not hdr or hdr == ['']: + continue + if hdr[0].startswith('##gff-version') and hdr[0].find('2') < 0: return False - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != 9: - return False + # The gff-version header comment may have been stripped, so inspect the data + if hdr[0].startswith('#'): + continue + if len(hdr) != 9: + return False + try: + int(hdr[3]) + int(hdr[4]) + except Exception: + return False + if hdr[5] != '.': try: - int(hdr[3]) - int(hdr[4]) + float(hdr[5]) except Exception: return False - if hdr[5] != '.': - try: - float(hdr[5]) - except Exception: - return False - if hdr[6] not in data.valid_strand: - return False - if hdr[7] not in self.valid_gff_frame: - return False - - # Check attributes for gene_id (transcript_id is also mandatory - # but not for genes) - attributes = parse_gff_attributes(hdr[8]) - if 'gene_id' not in attributes: - return False - return True + if hdr[6] not in data.valid_strand: + return False + if hdr[7] not in self.valid_gff_frame: + return False + # Check attributes for gene_id (transcript_id is also mandatory + # but not for genes) + attributes = parse_gff_attributes(hdr[8]) + if 'gene_id' not in attributes: + return False + found_valid_lines = True + return found_valid_lines except Exception: return False diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 00ce4513306..6a3657b05c5 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -235,24 +235,22 @@ def is_column_based(fname_or_file_prefix, sep='\t', skip=0): return False try: - headers = get_headers(fname_or_file_prefix, sep) + headers = get_headers(fname_or_file_prefix, sep, comment_designator='#')[skip:] except UnicodeDecodeError: return False count = 0 if not headers: return False - for hdr in headers[skip:]: - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) > 1: + for hdr in headers: + if hdr and hdr != ['']: + if count: + if len(hdr) != count: + return False + else: count = len(hdr) - break - if count < 2: - return False - for hdr in headers[skip:]: - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != count: - return False - return True + if count < 2: + return False + return count >= 2 def guess_ext(fname, sniff_order, is_binary=False): @@ -303,13 +301,13 @@ def guess_ext(fname, sniff_order, is_binary=False): >>> guess_ext(fname, sniff_order) 'gff3' >>> fname = get_test_fname('2.txt') - >>> guess_ext(fname, sniff_order) # 2.txt + >>> guess_ext(fname, sniff_order) 'txt' >>> fname = get_test_fname('2.tabular') >>> guess_ext(fname, sniff_order) 'tabular' >>> fname = get_test_fname('3.txt') - >>> guess_ext(fname, sniff_order) # 3.txt + >>> guess_ext(fname, sniff_order) 'txt' >>> fname = get_test_fname('test_tab1.tabular') >>> guess_ext(fname, sniff_order) @@ -454,6 +452,9 @@ def guess_ext(fname, sniff_order, is_binary=False): >>> fname = get_test_fname('1imzml') >>> guess_ext(fname, sniff_order) # This test case is ensuring doesn't throw exception, actual value could change if non-utf encoding handling improves. 'data' + >>> fname = get_test_fname('too_many_comments_gff3.tabular') + >>> guess_ext(fname, sniff_order) # It's a VCF but is sniffed as tabular because of the limit on the number of header lines we read + 'tabular' """ file_prefix = FilePrefix(fname) file_ext = run_sniffers_raw(file_prefix, sniff_order, is_binary) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 0dc4bd4e3a6..ec096762f5f 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -744,13 +744,12 @@ class BaseVcf(Tabular): def set_meta(self, dataset, **kwd): super().set_meta(dataset, **kwd) - source = open(dataset.file_name) - - # Skip comments. line = None - for line in source: - if not line.startswith('##'): - break + with compression_utils.get_fileobj(dataset.file_name) as fh: + # Skip comments. + for line in fh: + if not line.startswith('##'): + break if line and line.startswith('#'): # Found header line, get sample names. @@ -816,7 +815,7 @@ class VcfGz(BaseVcf, binary.Binary): return binascii.hexlify(last28) == b'1f8b08040000000000ff0600424302001b0003000000000000000000' def set_meta(self, dataset, **kwd): - super(BaseVcf, self).set_meta(dataset, **kwd) + super().set_meta(dataset, **kwd) """ Creates the index for the VCF file. """ # These metadata values are not accessible by users, always overwrite index_file = dataset.metadata.tabix_index diff --git a/lib/galaxy/datatypes/test/too_many_comments_gff3.tabular b/lib/galaxy/datatypes/test/too_many_comments_gff3.tabular new file mode 100644 index 00000000000..e3fb5113dc6 --- /dev/null +++ b/lib/galaxy/datatypes/test/too_many_comments_gff3.tabular @@ -0,0 +1,65 @@ +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +# +ctgA est match 5410 7503 . - . ID=EST:agt830.3;Target=agt830.3+1+595 +ctgA est HSP 7000 7503 . - . Parent=EST:agt830.3;Target=agt830.3+1+504 +ctgA est HSP 5410 5500 . ? . Parent=EST:agt830.3;Target=agt830.3+505+595 diff --git a/lib/galaxy/tools/parameters/wrapped_json.py b/lib/galaxy/tools/parameters/wrapped_json.py index f112bde13d1..d397783e082 100644 --- a/lib/galaxy/tools/parameters/wrapped_json.py +++ b/lib/galaxy/tools/parameters/wrapped_json.py @@ -100,10 +100,13 @@ def _json_wrap_input(input, value_wrapper, profile, handle_files="skip"): else: json_value = _cast_if_not_none(value_wrapper, bool, empty_to_none=input.optional) elif input_type == "select": - if input.multiple and packaging.version.parse(str(profile)) >= packaging.version.parse('20.05'): - json_value = [_ for _ in _cast_if_not_none(value_wrapper.value, list)] - else: + if packaging.version.parse(str(profile)) < packaging.version.parse('20.05'): json_value = _cast_if_not_none(value_wrapper, str) + else: + if input.multiple: + json_value = [str(_) for _ in _cast_if_not_none(value_wrapper.value, list)] + else: + json_value = _cast_if_not_none(value_wrapper.value, str) elif input_type == "data_column": # value is a SelectToolParameterWrapper() if input.multiple: diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 7420de8cadc..0d7ca77cb8a 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -257,7 +257,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn Lists all versions of this workflow. """ - stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id, **kwds) + instance = util.string_as_bool(kwds.get("instance", "false")) + stored_workflow = self.workflow_manager.get_stored_accessible_workflow(trans, workflow_id, by_stored_id=not instance) return [{'version': i, 'update_time': str(w.update_time), 'steps': len(w.steps)} for i, w in enumerate(reversed(stored_workflow.workflows))] @expose_api diff --git a/test/functional/tools/inputs_as_json.xml b/test/functional/tools/inputs_as_json.xml index 465aba5eda2..1aff602e796 100644 --- a/test/functional/tools/inputs_as_json.xml +++ b/test/functional/tools/inputs_as_json.xml @@ -27,8 +27,8 @@ if test_case == "1": assert_equals(as_dict["inttest"], 12456) assert_equals(as_dict["floattest"], 6.789) assert_equals(as_dict["radio_select"], "a_radio") - assert_equals(as_dict["optional_select"], None) - assert_equals(as_dict["optional_multiple_select"], []) + assert_equals(as_dict["optional_select"], "None") + assert_equals(as_dict["optional_multiple_select"], "None") assert_equals(as_dict["repeat"][0]["r"], "000000") assert_equals(as_dict["repeat"][1]["r"], "FFFFFF") assert_equals(as_dict["cond"]["more_text"], "fdefault") @@ -43,7 +43,7 @@ elif test_case == "2": assert_equals(as_dict["floattest"], 1.0) assert_equals(as_dict["radio_select"], "a_radio") assert_equals(as_dict["optional_select"], "a") - assert_equals(as_dict["optional_multiple_select"], ['a', 'b']) + assert_equals(as_dict["optional_multiple_select"], 'a,b') assert_equals(as_dict["repeat"][0]["r"], "000000") assert_equals(as_dict["cond"]["cond_test"], "second") assert_equals(as_dict["cond"]["more_text"], "sdefault") diff --git a/test/functional/tools/inputs_as_json_profile.xml b/test/functional/tools/inputs_as_json_profile.xml index 9fc04164878..162a6fa7345 100644 --- a/test/functional/tools/inputs_as_json_profile.xml +++ b/test/functional/tools/inputs_as_json_profile.xml @@ -1,7 +1,7 @@ - +