From 3d5fd7b5e27d33df02ea929f73250163f2e4e007 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 17 Aug 2012 11:35:31 -0400 Subject: [PATCH 01/16] Migrate NCBI Blast+ (and blastxml) to the toolshed. --- lib/galaxy/datatypes/registry.py | 3 - lib/galaxy/datatypes/xml.py | 120 ------------------ .../tool_shed/migrate/versions/0004_tools.py | 12 ++ scripts/migrate_tools/0004_tools.sh | 4 + scripts/migrate_tools/0004_tools.xml | 12 ++ 5 files changed, 28 insertions(+), 123 deletions(-) create mode 100644 lib/galaxy/tool_shed/migrate/versions/0004_tools.py create mode 100644 scripts/migrate_tools/0004_tools.sh create mode 100644 scripts/migrate_tools/0004_tools.xml diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 50315904e00..b054b423d11 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -276,7 +276,6 @@ class Registry( object ): 'axt' : sequence.Axt(), 'bam' : binary.Bam(), 'bed' : interval.Bed(), - 'blastxml' : xml.BlastXml(), 'coverage' : coverage.LastzCoverage(), 'customtrack' : interval.CustomTrack(), 'csfasta' : sequence.csFasta(), @@ -310,7 +309,6 @@ class Registry( object ): 'axt' : 'text/plain', 'bam' : 'application/octet-stream', 'bed' : 'text/plain', - 'blastxml' : 'application/xml', 'customtrack' : 'text/plain', 'csfasta' : 'text/plain', 'eland' : 'application/octet-stream', @@ -348,7 +346,6 @@ class Registry( object ): self.sniff_order = [ binary.Bam(), binary.Sff(), - xml.BlastXml(), xml.GenericXml(), sequence.Maf(), sequence.Lav(), diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index ef1f1623e28..37f34f55169 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -27,9 +27,6 @@ class GenericXml( data.Text ): >>> fname = get_test_fname( 'megablast_xml_parser_test1.blastxml' ) >>> GenericXml().sniff( fname ) True - >>> fname = get_test_fname( 'tblastn_four_human_vs_rhodopsin.xml' ) - >>> BlastXml().sniff( fname ) - True >>> fname = get_test_fname( 'interval.interval' ) >>> GenericXml().sniff( fname ) False @@ -50,123 +47,6 @@ class GenericXml( data.Text ): data.Text.merge(split_files, output_file) merge = staticmethod(merge) -class BlastXml( GenericXml ): - """NCBI Blast XML Output data""" - file_ext = "blastxml" - - def set_peek( self, dataset, is_multi_byte=False ): - """Set the peek and blurb text""" - if not dataset.dataset.purged: - dataset.peek = data.get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) - dataset.blurb = 'NCBI Blast XML data' - else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' - def sniff( self, filename ): - """ - Determines whether the file is blastxml - - >>> fname = get_test_fname( 'megablast_xml_parser_test1.blastxml' ) - >>> BlastXml().sniff( fname ) - True - >>> fname = get_test_fname( 'tblastn_four_human_vs_rhodopsin.xml' ) - >>> BlastXml().sniff( fname ) - True - >>> fname = get_test_fname( 'interval.interval' ) - >>> BlastXml().sniff( fname ) - False - """ - #TODO - Use a context manager on Python 2.5+ to close handle - handle = open(filename) - line = handle.readline() - if line.strip() != '': - handle.close() - return False - line = handle.readline() - if line.strip() not in ['', - '']: - handle.close() - return False - line = handle.readline() - if line.strip() != '': - handle.close() - return False - handle.close() - return True - - def merge(split_files, output_file): - """Merging multiple XML files is non-trivial and must be done in subclasses.""" - if len(split_files) == 1: - #For one file only, use base class method (move/copy) - return data.Text.merge(split_files, output_file) - out = open(output_file, "w") - h = None - for f in split_files: - h = open(f) - body = False - header = h.readline() - if not header: - out.close() - h.close() - raise ValueError("BLAST XML file %s was empty" % f) - if header.strip() != '': - out.write(header) #for diagnosis - out.close() - h.close() - raise ValueError("%s is not an XML file!" % f) - line = h.readline() - header += line - if line.strip() not in ['', - '']: - out.write(header) #for diagnosis - out.close() - h.close() - raise ValueError("%s is not a BLAST XML file!" % f) - while True: - line = h.readline() - if not line: - out.write(header) #for diagnosis - out.close() - h.close() - raise ValueError("BLAST XML file %s ended prematurely" % f) - header += line - if "" in line: - break - if len(header) > 10000: - #Something has gone wrong, don't load too much into memory! - #Write what we have to the merged file for diagnostics - out.write(header) - out.close() - h.close() - raise ValueError("BLAST XML file %s has too long a header!" % f) - if "" not in header: - out.close() - h.close() - raise ValueError("%s is not a BLAST XML file:\n%s\n..." % (f, header)) - if f == split_files[0]: - out.write(header) - old_header = header - elif old_header[:300] != header[:300]: - #Enough to check and match - out.close() - h.close() - raise ValueError("BLAST XML headers don't match for %s and %s - have:\n%s\n...\n\nAnd:\n%s\n...\n" \ - % (split_files[0], f, old_header[:300], header[:300])) - else: - out.write(" \n") - for line in h: - if "" in line: - break - #TODO - Increment and if required automatic query names - #like Query_3 to be increasing? - out.write(line) - h.close() - out.write(" \n") - out.write("\n") - out.close() - merge = staticmethod(merge) - - class MEMEXml( GenericXml ): """MEME XML Output data""" file_ext = "memexml" diff --git a/lib/galaxy/tool_shed/migrate/versions/0004_tools.py b/lib/galaxy/tool_shed/migrate/versions/0004_tools.py new file mode 100644 index 00000000000..f20bd83b251 --- /dev/null +++ b/lib/galaxy/tool_shed/migrate/versions/0004_tools.py @@ -0,0 +1,12 @@ +""" +The NCBI BLAST+ tools have been eliminated from the distribution. The tools and datatypes are are now available in repositories named ncbi_blast_plus +and blast_datatypes, respectively, from the main Galaxy tool shed at http://toolshed.g2.bx.psu.edu will be installed into your local Galaxy instance +at the location discussed above by running the following command. +""" + +import sys + +def upgrade(): + print __doc__ +def downgrade(): + pass diff --git a/scripts/migrate_tools/0004_tools.sh b/scripts/migrate_tools/0004_tools.sh new file mode 100644 index 00000000000..40b76956fa2 --- /dev/null +++ b/scripts/migrate_tools/0004_tools.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd `dirname $0`/../.. +python ./scripts/migrate_tools/migrate_tools.py 0004_tools.xml $@ diff --git a/scripts/migrate_tools/0004_tools.xml b/scripts/migrate_tools/0004_tools.xml new file mode 100644 index 00000000000..53664f9823c --- /dev/null +++ b/scripts/migrate_tools/0004_tools.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + From 97e69fbbdfd820b3044ef3289484850bf3faa566 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Fri, 17 Aug 2012 13:55:37 -0400 Subject: [PATCH 02/16] Fix chunk-serving logic for tabular files. --- lib/galaxy/datatypes/tabular.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index ce3b043b658..0939edae159 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -264,10 +264,10 @@ class Tabular( data.Text ): def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None): #TODO Prevent failure when displaying extremely long > 50kb lines. - if to_ext or not preview: - return self._serve_raw(trans, dataset, to_ext) if chunk: return self.get_chunk(trans, dataset, chunk) + if to_ext or not preview: + return self._serve_raw(trans, dataset, to_ext) else: column_names = 'null' if dataset.metadata.column_names: @@ -644,4 +644,5 @@ class FeatureLocationIndex( Tabular ): """ file_ext='fli' MetadataElement( name="columns", default=2, desc="Number of columns", readonly=True, visible=False ) - MetadataElement( name="column_types", default=['str', 'str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False, no_value=[] ) \ No newline at end of file + MetadataElement( name="column_types", default=['str', 'str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False, no_value=[] ) + From 3360fe30725e8aeb45595a44936470753d264811 Mon Sep 17 00:00:00 2001 From: Carl Eberhard Date: Fri, 17 Aug 2012 14:39:32 -0400 Subject: [PATCH 03/16] new data for bwa_wrapper/bwa_color_wrapper functional tests From dbc2c27a82c11d6693b92c9dd8358879390cfca8 Mon Sep 17 00:00:00 2001 From: Scott McManus Date: Fri, 17 Aug 2012 15:15:48 -0400 Subject: [PATCH 04/16] Minor tweak to exit code handling --- lib/galaxy/jobs/__init__.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 5ac63f9957b..50c185834c2 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -490,7 +490,6 @@ class JobWrapper( object ): if stderr contains anything, then False is returned. Note that the job id is just for messages. """ - err_msg = "" # By default, the tool succeeded. This covers the case where the code # has a bug but the tool was ok, and it lets a workflow continue. success = True @@ -507,7 +506,7 @@ class JobWrapper( object ): # Check the exit code ranges in the order in which # they were specified. Each exit_code is a StdioExitCode # that includes an applicable range. If the exit code was in - # that range, then apply the error level and add in a message. + # that range, then apply the error level and add a message. # If we've reached a fatal error rule, then stop. max_error_level = galaxy.tools.StdioErrorLevel.NO_ERROR for stdio_exit_code in self.tool.stdio_exit_codes: @@ -515,20 +514,16 @@ class JobWrapper( object ): tool_exit_code <= stdio_exit_code.range_end ): # Tack on a generic description of the code # plus a specific code description. For example, - # this might append "Job 42: Warning: Out of Memory\n". - # TODO: Find somewhere to stick the err_msg - - # possibly to the source (stderr/stdout), possibly - # in a new db column. + # this might prepend "Job 42: Warning: Out of Memory\n". code_desc = stdio_exit_code.desc if ( None == code_desc ): code_desc = "" - tool_msg = ( "Job %s: %s: Exit code %d: %s" % ( - job.get_id_tag(), - galaxy.tools.StdioErrorLevel.desc( tool_exit_code ), + tool_msg = ( "%s: Exit code %d: %s" % ( + galaxy.tools.StdioErrorLevel.desc( stdio_exit_code.error_level ), tool_exit_code, code_desc ) ) - log.info( tool_msg ) - stderr = err_msg + stderr + log.info( "Job %s: %s" % (job.get_id_tag(), tool_msg) ) + stderr = tool_msg + "\n" + stderr max_error_level = max( max_error_level, stdio_exit_code.error_level ) if ( max_error_level >= @@ -571,7 +566,6 @@ class JobWrapper( object ): re.IGNORECASE ) if ( regex_match ): rexmsg = self.regex_err_msg( regex_match, regex) - # DELETEME log.info( "Job %s: %s" % ( job.get_id_tag(), rexmsg ) ) stderr = rexmsg + "\n" + stderr From 860bb2673c6a0324a6d0dc4738fe632d2e1a458b Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Fri, 17 Aug 2012 17:49:41 -0400 Subject: [PATCH 05/16] Converters enhancements: (a) ignore track lines in wiggle to bigwig converter and add clip option and (b) enable visualization of bedgraph datasets via a bedgraph to bigwig converter. --- .../converters/bedgraph_to_bigwig_converter.xml | 14 ++++++++++++++ .../converters/wig_to_bigwig_converter.xml | 2 +- lib/galaxy/datatypes/interval.py | 5 +++-- 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml new file mode 100644 index 00000000000..77d2c2a42e6 --- /dev/null +++ b/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml @@ -0,0 +1,14 @@ + \ No newline at end of file diff --git a/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml index e85c3a7b138..d90702efe78 100644 --- a/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml +++ b/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml @@ -1,6 +1,6 @@