Merge branch 'dev' of https://github.com/galaxyproject/galaxy into workflow_form_002

CONTRIBUTING.md

@@ -50,13 +50,12 @@ the [development mailing list](http://dev.list.galaxyproject.org/).
   * Most bug fixes to previously release components (things in galaxy-dist)
     should be made against the recent `release_XX.XX` branch (`git checkout release_XX.XX`).
 
-  * Serious security problems should not be fixed via pull request -
-    please responsibly disclose these by e-mailing them (with or
-    without patches) to galaxy-lab@bx.psu.edu. The Galaxy core
-    development team will issue patches to public servers before
-    announcing the issue to ensure there is time to patch and
-    highlight these fixes widely. We will provide you credit for the
-    discovery when publicly disclosing the issue.
+  * Serious security problems should not be fixed via pull request - please
+    responsibly disclose these by e-mailing them (with or without patches) to
+    galaxy-lab@lists.galaxyproject.org. The Galaxy core development team will
+    issue patches to public servers before announcing the issue to ensure there
+    is time to patch and highlight these fixes widely. We will provide you
+    credit for the discovery when publicly disclosing the issue.
 
 * If your changes modify code - please ensure the resulting files
   conform to Galaxy [style

config/datatypes_conf.xml.sample

@@ -51,7 +51,7 @@
       <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/>
     </datatype>
     <!-- MSI added Datatypes -->
-    <datatype extension="csv" type="galaxy.datatypes.tabular:Tabular" subclass="True" display_in_upload="true" /> <!-- FIXME: csv is 'tabular'ized data, but not 'tab-delimited'; the class used here is intended for 'tab-delimited' -->
+    <datatype extension="csv" type="galaxy.datatypes.tabular:CSV" display_in_upload="true" />
     <!-- End MSI added Datatypes -->
     <datatype extension="customtrack" type="galaxy.datatypes.interval:CustomTrack"/>
     <datatype extension="bowtie_color_index" type="galaxy.datatypes.ngsindex:BowtieColorIndex" mimetype="text/html" display_in_upload="False"/>
@@ -315,6 +315,7 @@
     <sniffer type="galaxy.datatypes.interval:Gff"/>
     <sniffer type="galaxy.datatypes.interval:Gff3"/>
     <sniffer type="galaxy.datatypes.tabular:Pileup"/>
+    <sniffer type="galaxy.datatypes.tabular:CSV"/>
     <sniffer type="galaxy.datatypes.interval:Interval"/>
     <sniffer type="galaxy.datatypes.tabular:Sam"/>
     <sniffer type="galaxy.datatypes.data:Newick"/>

config/plugins/visualizations/charts/config/charts.xml

@@ -5,6 +5,7 @@
         <data_source>
             <model_class>HistoryDatasetAssociation</model_class>
             <test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test>
+            <test type="isinstance" test_attr="datatype" result_type="datatype">tabular.CSV</test>
             <to_param param_attr="id">dataset_id</to_param>
         </data_source>
     </data_sources>

config/plugins/visualizations/scatterplot/config/scatterplot.xml

@@ -5,6 +5,7 @@
         <data_source>
             <model_class>HistoryDatasetAssociation</model_class>
             <test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test>
+            <test type="isinstance" test_attr="datatype" result_type="datatype">tabular.CSV</test>
             <to_param param_attr="id">dataset_id</to_param>
         </data_source>
     </data_sources>

lib/galaxy/datatypes/registry.py

@@ -695,6 +695,7 @@ class Registry( object ):
                 'scf'         : binary.Scf(),
                 'sff'         : binary.Sff(),
                 'tabular'     : tabular.Tabular(),
+                'csv'         : tabular.CSV(),
                 'taxonomy'    : tabular.Taxonomy(),
                 'txt'         : data.Text(),
                 'wig'         : interval.Wiggle(),
@@ -727,6 +728,7 @@ class Registry( object ):
                 'scf'         : 'application/octet-stream',
                 'sff'         : 'application/octet-stream',
                 'tabular'     : 'text/plain',
+                'csv'         : 'text/plain',
                 'taxonomy'    : 'text/plain',
                 'txt'         : 'text/plain',
                 'wig'         : 'text/plain',
@@ -761,7 +763,8 @@ class Registry( object ):
                 tabular.Pileup(),
                 interval.Interval(),
                 tabular.Sam(),
-                tabular.Eland()
+                tabular.Eland(),
+                tabular.CSV()
             ]
 
     def get_converters_by_datatype( self, ext ):

lib/galaxy/datatypes/tabular.py

@@ -7,6 +7,7 @@ pkg_resources.require( "bx-python" )
 import gzip
 import logging
 import os
+import csv
 from cgi import escape
 from galaxy import util
 from galaxy.datatypes import data
@@ -19,159 +20,90 @@ import dataproviders
 
 log = logging.getLogger(__name__)
 
+
 @dataproviders.decorators.has_dataproviders
-class Tabular( data.Text ):
-    """Tab delimited data"""
+class TabularData( data.Text ):
+    """Generic tabular data"""
     edam_format = "format_3475"
     # All tabular data is chunkable.
     CHUNKABLE = True
 
     """Add metadata elements"""
     MetadataElement( name="comment_lines", default=0, desc="Number of comment lines", readonly=False, optional=True, no_value=0 )
+    MetadataElement( name="data_lines", default=0, desc="Number of data lines", readonly=True, visible=False, optional=True, no_value=0 )
     MetadataElement( name="columns", default=0, desc="Number of columns", readonly=True, visible=False, no_value=0 )
     MetadataElement( name="column_types", default=[], desc="Column types", param=metadata.ColumnTypesParameter, readonly=True, visible=False, no_value=[] )
     MetadataElement( name="column_names", default=[], desc="Column names", readonly=True, visible=False, optional=True, no_value=[] )
+    MetadataElement( name="delimiter", default='\t', desc="Data delimiter", readonly=True, visible=False, optional=True, no_value=[] )
 
-    def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 100000, max_guess_type_data_lines = None, **kwd ):
-        """
-        Tries to determine the number of columns as well as those columns that
-        contain numerical values in the dataset.  A skip parameter is used
-        because various tabular data types reuse this function, and their data
-        type classes are responsible to determine how many invalid comment
-        lines should be skipped. Using None for skip will cause skip to be
-        zero, but the first line will be processed as a header. A
-        max_data_lines parameter is used because various tabular data types
-        reuse this function, and their data type classes are responsible to
-        determine how many data lines should be processed to ensure that the
-        non-optional metadata parameters are properly set; if used, optional
-        metadata parameters will be set to None, unless the entire file has
-        already been read. Using None for max_data_lines will process all data
-        lines.
+    def set_peek( self, dataset, line_count=None, is_multi_byte=False):
+        super(TabularData, self).set_peek( dataset, line_count=line_count, is_multi_byte=is_multi_byte)
+        if dataset.metadata.comment_lines:
+            dataset.blurb = "%s, %s comments" % ( dataset.blurb, util.commaify( str( dataset.metadata.comment_lines ) ) )
 
-        Items of interest:
+    def displayable( self, dataset ):
+        try:
+            return dataset.has_data() \
+                and dataset.state == dataset.states.OK \
+                and dataset.metadata.columns > 0 \
+                and dataset.metadata.data_lines != 0
+        except:
+            return False
 
-        1. We treat 'overwrite' as always True (we always want to set tabular metadata when called).
-        2. If a tabular file has no data, it will have one column of type 'str'.
-        3. We used to check only the first 100 lines when setting metadata and this class's
-           set_peek() method read the entire file to determine the number of lines in the file.
-           Since metadata can now be processed on cluster nodes, we've merged the line count portion
-           of the set_peek() processing here, and we now check the entire contents of the file.
-        """
-        # Store original skip value to check with later
-        requested_skip = skip
-        if skip is None:
-            skip = 0
-        column_type_set_order = [ 'int', 'float', 'list', 'str'  ] #Order to set column types in
-        default_column_type = column_type_set_order[-1] # Default column type is lowest in list
-        column_type_compare_order = list( column_type_set_order ) #Order to compare column types
-        column_type_compare_order.reverse()
-        def type_overrules_type( column_type1, column_type2 ):
-            if column_type1 is None or column_type1 == column_type2:
-                return False
-            if column_type2 is None:
-                return True
-            for column_type in column_type_compare_order:
-                if column_type1 == column_type:
-                    return True
-                if column_type2 == column_type:
-                    return False
-            #neither column type was found in our ordered list, this cannot happen
-            raise "Tried to compare unknown column types"
-        def is_int( column_text ):
-            try:
-                int( column_text )
-                return True
-            except:
-                return False
-        def is_float( column_text ):
-            try:
-                float( column_text )
-                return True
-            except:
-                if column_text.strip().lower() == 'na':
-                    return True #na is special cased to be a float
-                return False
-        def is_list( column_text ):
-            return "," in column_text
-        def is_str( column_text ):
-            #anything, except an empty string, is True
-            if column_text == "":
-                return False
-            return True
-        is_column_type = {} #Dict to store column type string to checking function
-        for column_type in column_type_set_order:
-            is_column_type[column_type] = locals()[ "is_%s" % ( column_type ) ]
-        def guess_column_type( column_text ):
-            for column_type in column_type_set_order:
-                if is_column_type[column_type]( column_text ):
-                    return column_type
-            return None
-        data_lines = 0
-        comment_lines = 0
-        column_types = []
-        first_line_column_types = [default_column_type] # default value is one column of type str
-        if dataset.has_data():
-            #NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default
-            dataset_fh = open( dataset.file_name )
-            i = 0
-            while True:
-                line = dataset_fh.readline()
-                if not line: break
-                line = line.rstrip( '\r\n' )
-                if i < skip or not line or line.startswith( '#' ):
-                    # We'll call blank lines comments
-                    comment_lines += 1
-                else:
-                    data_lines += 1
-                    if max_guess_type_data_lines is None or data_lines <= max_guess_type_data_lines:
-                        fields = line.split( '\t' )
-                        for field_count, field in enumerate( fields ):
-                            if field_count >= len( column_types ): #found a previously unknown column, we append None
-                                column_types.append( None )
-                            column_type = guess_column_type( field )
-                            if type_overrules_type( column_type, column_types[field_count] ):
-                                column_types[field_count] = column_type
-                    if i == 0 and requested_skip is None:
-                        # This is our first line, people seem to like to upload files that have a header line, but do not
-                        # start with '#' (i.e. all column types would then most likely be detected as str).  We will assume
-                        # that the first line is always a header (this was previous behavior - it was always skipped).  When
-                        # the requested skip is None, we only use the data from the first line if we have no other data for
-                        # a column.  This is far from perfect, as
-                        # 1,2,3	1.1	2.2	qwerty
-                        # 0	0		1,2,3
-                        # will be detected as
-                        # "column_types": ["int", "int", "float", "list"]
-                        # instead of
-                        # "column_types": ["list", "float", "float", "str"]  *** would seem to be the 'Truth' by manual
-                        # observation that the first line should be included as data.  The old method would have detected as
-                        # "column_types": ["int", "int", "str", "list"]
-                        first_line_column_types = column_types
-                        column_types = [ None for col in first_line_column_types ]
-                if max_data_lines is not None and data_lines >= max_data_lines:
-                    if dataset_fh.tell() != dataset.get_size():
-                        data_lines = None #Clear optional data_lines metadata value
-                        comment_lines = None #Clear optional comment_lines metadata value; additional comment lines could appear below this point
-                    break
-                i += 1
-            dataset_fh.close()
+    def get_chunk(self, trans, dataset, chunk):
+        ck_index = int(chunk)
+        f = open(dataset.file_name)
+        f.seek(ck_index * trans.app.config.display_chunk_size)
+        # If we aren't at the start of the file, seek to next newline.  Do this better eventually.
+        if f.tell() != 0:
+            cursor = f.read(1)
+            while cursor and cursor != '\n':
+                cursor = f.read(1)
+        ck_data = f.read(trans.app.config.display_chunk_size)
+        cursor = f.read(1)
+        while cursor and ck_data[-1] != '\n':
+            ck_data += cursor
+            cursor = f.read(1)
+        return dumps( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } )
+
+    def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd):
+        preview = util.string_as_bool( preview )
+        if chunk:
+            return self.get_chunk(trans, dataset, chunk)
+        elif to_ext or not preview:
+            to_ext = to_ext or dataset.extension
+            return self._serve_raw(trans, dataset, to_ext)
+        elif dataset.metadata.columns > 50:
+            # Fancy tabular display is only suitable for datasets without an incredibly large number of columns.
+            # We should add a new datatype 'matrix', with its own draw method, suitable for this kind of data.
+            # For now, default to the old behavior, ugly as it is.  Remove this after adding 'matrix'.
+            max_peek_size = 1000000  # 1 MB
+            if os.stat( dataset.file_name ).st_size < max_peek_size:
+                return open( dataset.file_name )
+            else:
+                trans.response.set_content_type( "text/html" )
+                return trans.stream_template_mako( "/dataset/large_file.mako",
+                                            truncated_data=open( dataset.file_name ).read(max_peek_size),
+                                            data=dataset)
+        else:
+            column_names = 'null'
+            if dataset.metadata.column_names:
+                column_names = dataset.metadata.column_names
+            elif hasattr(dataset.datatype, 'column_names'):
+                column_names = dataset.datatype.column_names
+            column_types = dataset.metadata.column_types
+            if not column_types:
+                column_types = []
+            column_number = dataset.metadata.columns
+            if column_number is None:
+                column_number = 'null'
+            return trans.fill_template( "/dataset/tabular_chunked.mako",
+                        dataset=dataset,
+                        chunk=self.get_chunk(trans, dataset, 0),
+                        column_number=column_number,
+                        column_names=column_names,
+                        column_types=column_types )
 
-        #we error on the larger number of columns
-        #first we pad our column_types by using data from first line
-        if len( first_line_column_types ) > len( column_types ):
-            for column_type in first_line_column_types[len( column_types ):]:
-                column_types.append( column_type )
-        #Now we fill any unknown (None) column_types with data from first line
-        for i in range( len( column_types ) ):
-            if column_types[i] is None:
-                if len( first_line_column_types ) <= i or first_line_column_types[i] is None:
-                    column_types[i] = default_column_type
-                else:
-                    column_types[i] = first_line_column_types[i]
-        # Set the discovered metadata values for the dataset
-        dataset.metadata.data_lines = data_lines
-        dataset.metadata.comment_lines = comment_lines
-        dataset.metadata.column_types = column_types
-        dataset.metadata.columns = len( column_types )
     def make_html_table( self, dataset, **kwargs ):
         """Create HTML table, used for displaying peek"""
         out = ['<table cellspacing="0" cellpadding="3">']
@@ -227,7 +159,7 @@ class Tabular( data.Text ):
             out.append( '</tr>' )
         except Exception, exc:
             log.exception( 'make_html_peek_header failed on HDA %s' % dataset.id )
-            raise Exception, "Can't create peek header %s" % str( exc )
+            raise Exception( "Can't create peek header %s" % str( exc ) )
         return "".join( out )
 
     def make_html_peek_rows( self, dataset, skipchars=None, **kwargs ):
@@ -244,7 +176,7 @@ class Tabular( data.Text ):
                 if line.startswith( tuple( skipchars ) ):
                     out.append( '<tr><td colspan="100%%">%s</td></tr>' % escape( line ) )
                 elif line:
-                    elems = line.split( '\t' )
+                    elems = line.split( dataset.metadata.delimiter )
                     # we may have an invalid comment line or invalid data
                     if len( elems ) != columns:
                         out.append( '<tr><td colspan="100%%">%s</td></tr>' % escape( line ) )
@@ -255,106 +187,200 @@ class Tabular( data.Text ):
                         out.append( '</tr>' )
         except Exception, exc:
             log.exception( 'make_html_peek_rows failed on HDA %s' % dataset.id )
-            raise Exception, "Can't create peek rows %s" % str( exc )
+            raise Exception( "Can't create peek rows %s" % str( exc ) )
         return "".join( out )
 
-    def get_chunk(self, trans, dataset, chunk):
-        ck_index = int(chunk)
-        f = open(dataset.file_name)
-        f.seek(ck_index * trans.app.config.display_chunk_size)
-        # If we aren't at the start of the file, seek to next newline.  Do this better eventually.
-        if f.tell() != 0:
-            cursor = f.read(1)
-            while cursor and cursor != '\n':
-                cursor = f.read(1)
-        ck_data = f.read(trans.app.config.display_chunk_size)
-        cursor = f.read(1)
-        while cursor and ck_data[-1] != '\n':
-            ck_data += cursor
-            cursor = f.read(1)
-        return dumps( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } )
-
-    def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd):
-        preview = util.string_as_bool( preview )
-        if chunk:
-            return self.get_chunk(trans, dataset, chunk)
-        elif to_ext or not preview:
-            to_ext = to_ext or dataset.extension
-            return self._serve_raw(trans, dataset, to_ext)
-        elif dataset.metadata.columns > 50:
-            #Fancy tabular display is only suitable for datasets without an incredibly large number of columns.
-            #We should add a new datatype 'matrix', with its own draw method, suitable for this kind of data.
-            #For now, default to the old behavior, ugly as it is.  Remove this after adding 'matrix'.
-            max_peek_size = 1000000 # 1 MB
-            if os.stat( dataset.file_name ).st_size < max_peek_size:
-                return open( dataset.file_name )
-            else:
-                trans.response.set_content_type( "text/html" )
-                return trans.stream_template_mako( "/dataset/large_file.mako",
-                                            truncated_data = open( dataset.file_name ).read(max_peek_size),
-                                            data = dataset)
-        else:
-            column_names = 'null'
-            if dataset.metadata.column_names:
-                column_names = dataset.metadata.column_names
-            elif hasattr(dataset.datatype, 'column_names'):
-                column_names = dataset.datatype.column_names
-            column_types = dataset.metadata.column_types
-            if not column_types:
-                column_types = []
-            column_number = dataset.metadata.columns
-            if column_number is None:
-                column_number = 'null'
-            return trans.fill_template( "/dataset/tabular_chunked.mako",
-                        dataset = dataset,
-                        chunk = self.get_chunk(trans, dataset, 0),
-                        column_number = column_number,
-                        column_names = column_names,
-                        column_types = column_types )
-
-    def set_peek( self, dataset, line_count=None, is_multi_byte=False):
-        super(Tabular, self).set_peek( dataset, line_count=line_count, is_multi_byte=is_multi_byte)
-        if dataset.metadata.comment_lines:
-            dataset.blurb = "%s, %s comments" % ( dataset.blurb, util.commaify( str( dataset.metadata.comment_lines ) ) )
     def display_peek( self, dataset ):
         """Returns formatted html of peek"""
         return self.make_html_table( dataset )
-    def displayable( self, dataset ):
-        try:
-            return dataset.has_data() \
-                and dataset.state == dataset.states.OK \
-                and dataset.metadata.columns > 0 \
-                and dataset.metadata.data_lines != 0
-        except:
-            return False
-    def as_gbrowse_display_file( self, dataset, **kwd ):
-        return open( dataset.file_name )
-    def as_ucsc_display_file( self, dataset, **kwd ):
-        return open( dataset.file_name )
 
     # ------------- Dataproviders
     @dataproviders.decorators.dataprovider_factory( 'column', dataproviders.column.ColumnarDataProvider.settings )
     def column_dataprovider( self, dataset, **settings ):
         """Uses column settings that are passed in"""
         dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
-        return dataproviders.column.ColumnarDataProvider( dataset_source, **settings )
+        delimiter = dataset.metadata.delimiter
+        return dataproviders.column.ColumnarDataProvider( dataset_source, deliminator=delimiter, **settings )
 
     @dataproviders.decorators.dataprovider_factory( 'dataset-column',
                                                     dataproviders.column.ColumnarDataProvider.settings )
     def dataset_column_dataprovider( self, dataset, **settings ):
         """Attempts to get column settings from dataset.metadata"""
-        return dataproviders.dataset.DatasetColumnarDataProvider( dataset, **settings )
+        delimiter = dataset.metadata.delimiter
+        return dataproviders.dataset.DatasetColumnarDataProvider( dataset, deliminator=delimiter, **settings )
 
     @dataproviders.decorators.dataprovider_factory( 'dict', dataproviders.column.DictDataProvider.settings )
     def dict_dataprovider( self, dataset, **settings ):
         """Uses column settings that are passed in"""
         dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
-        return dataproviders.column.DictDataProvider( dataset_source, **settings )
+        delimiter = dataset.metadata.delimiter
+        return dataproviders.column.DictDataProvider( dataset_source, deliminator=delimiter, **settings )
 
     @dataproviders.decorators.dataprovider_factory( 'dataset-dict', dataproviders.column.DictDataProvider.settings )
     def dataset_dict_dataprovider( self, dataset, **settings ):
         """Attempts to get column settings from dataset.metadata"""
-        return dataproviders.dataset.DatasetDictDataProvider( dataset, **settings )
+        delimiter = dataset.metadata.delimiter
+        return dataproviders.dataset.DatasetDictDataProvider( dataset, deliminator=delimiter, **settings )
+
+
+@dataproviders.decorators.has_dataproviders
+class Tabular( TabularData ):
+    """Tab delimited data"""
+
+    def set_meta( self, dataset, overwrite=True, skip=None, max_data_lines=100000, max_guess_type_data_lines=None, **kwd ):
+        """
+        Tries to determine the number of columns as well as those columns that
+        contain numerical values in the dataset.  A skip parameter is used
+        because various tabular data types reuse this function, and their data
+        type classes are responsible to determine how many invalid comment
+        lines should be skipped. Using None for skip will cause skip to be
+        zero, but the first line will be processed as a header. A
+        max_data_lines parameter is used because various tabular data types
+        reuse this function, and their data type classes are responsible to
+        determine how many data lines should be processed to ensure that the
+        non-optional metadata parameters are properly set; if used, optional
+        metadata parameters will be set to None, unless the entire file has
+        already been read. Using None for max_data_lines will process all data
+        lines.
+
+        Items of interest:
+
+        1. We treat 'overwrite' as always True (we always want to set tabular metadata when called).
+        2. If a tabular file has no data, it will have one column of type 'str'.
+        3. We used to check only the first 100 lines when setting metadata and this class's
+           set_peek() method read the entire file to determine the number of lines in the file.
+           Since metadata can now be processed on cluster nodes, we've merged the line count portion
+           of the set_peek() processing here, and we now check the entire contents of the file.
+        """
+        # Store original skip value to check with later
+        requested_skip = skip
+        if skip is None:
+            skip = 0
+        column_type_set_order = [ 'int', 'float', 'list', 'str'  ]  # Order to set column types in
+        default_column_type = column_type_set_order[-1]  # Default column type is lowest in list
+        column_type_compare_order = list( column_type_set_order )  # Order to compare column types
+        column_type_compare_order.reverse()
+
+        def type_overrules_type( column_type1, column_type2 ):
+            if column_type1 is None or column_type1 == column_type2:
+                return False
+            if column_type2 is None:
+                return True
+            for column_type in column_type_compare_order:
+                if column_type1 == column_type:
+                    return True
+                if column_type2 == column_type:
+                    return False
+            # neither column type was found in our ordered list, this cannot happen
+            raise "Tried to compare unknown column types"
+
+        def is_int( column_text ):
+            try:
+                int( column_text )
+                return True
+            except:
+                return False
+
+        def is_float( column_text ):
+            try:
+                float( column_text )
+                return True
+            except:
+                if column_text.strip().lower() == 'na':
+                    return True  # na is special cased to be a float
+                return False
+
+        def is_list( column_text ):
+            return "," in column_text
+
+        def is_str( column_text ):
+            # anything, except an empty string, is True
+            if column_text == "":
+                return False
+            return True
+        is_column_type = {}  # Dict to store column type string to checking function
+        for column_type in column_type_set_order:
+            is_column_type[column_type] = locals()[ "is_%s" % ( column_type ) ]
+
+        def guess_column_type( column_text ):
+            for column_type in column_type_set_order:
+                if is_column_type[column_type]( column_text ):
+                    return column_type
+            return None
+        data_lines = 0
+        comment_lines = 0
+        column_types = []
+        first_line_column_types = [default_column_type]  # default value is one column of type str
+        if dataset.has_data():
+            # NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default
+            dataset_fh = open( dataset.file_name )
+            i = 0
+            while True:
+                line = dataset_fh.readline()
+                if not line:
+                    break
+                line = line.rstrip( '\r\n' )
+                if i < skip or not line or line.startswith( '#' ):
+                    # We'll call blank lines comments
+                    comment_lines += 1
+                else:
+                    data_lines += 1
+                    if max_guess_type_data_lines is None or data_lines <= max_guess_type_data_lines:
+                        fields = line.split( '\t' )
+                        for field_count, field in enumerate( fields ):
+                            if field_count >= len( column_types ):  # found a previously unknown column, we append None
+                                column_types.append( None )
+                            column_type = guess_column_type( field )
+                            if type_overrules_type( column_type, column_types[field_count] ):
+                                column_types[field_count] = column_type
+                    if i == 0 and requested_skip is None:
+                        # This is our first line, people seem to like to upload files that have a header line, but do not
+                        # start with '#' (i.e. all column types would then most likely be detected as str).  We will assume
+                        # that the first line is always a header (this was previous behavior - it was always skipped).  When
+                        # the requested skip is None, we only use the data from the first line if we have no other data for
+                        # a column.  This is far from perfect, as
+                        # 1,2,3	1.1	2.2	qwerty
+                        # 0	0		1,2,3
+                        # will be detected as
+                        # "column_types": ["int", "int", "float", "list"]
+                        # instead of
+                        # "column_types": ["list", "float", "float", "str"]  *** would seem to be the 'Truth' by manual
+                        # observation that the first line should be included as data.  The old method would have detected as
+                        # "column_types": ["int", "int", "str", "list"]
+                        first_line_column_types = column_types
+                        column_types = [ None for col in first_line_column_types ]
+                if max_data_lines is not None and data_lines >= max_data_lines:
+                    if dataset_fh.tell() != dataset.get_size():
+                        data_lines = None  # Clear optional data_lines metadata value
+                        comment_lines = None  # Clear optional comment_lines metadata value; additional comment lines could appear below this point
+                    break
+                i += 1
+            dataset_fh.close()
+
+        # we error on the larger number of columns
+        # first we pad our column_types by using data from first line
+        if len( first_line_column_types ) > len( column_types ):
+            for column_type in first_line_column_types[len( column_types ):]:
+                column_types.append( column_type )
+        # Now we fill any unknown (None) column_types with data from first line
+        for i in range( len( column_types ) ):
+            if column_types[i] is None:
+                if len( first_line_column_types ) <= i or first_line_column_types[i] is None:
+                    column_types[i] = default_column_type
+                else:
+                    column_types[i] = first_line_column_types[i]
+        # Set the discovered metadata values for the dataset
+        dataset.metadata.data_lines = data_lines
+        dataset.metadata.comment_lines = comment_lines
+        dataset.metadata.column_types = column_types
+        dataset.metadata.columns = len( column_types )
+        dataset.metadata.delimiter = '\t'
+
+    def as_gbrowse_display_file( self, dataset, **kwd ):
+        return open( dataset.file_name )
+
+    def as_ucsc_display_file( self, dataset, **kwd ):
+        return open( dataset.file_name )
 
 
 class Taxonomy( Tabular ):
@@ -366,6 +392,7 @@ class Taxonomy( Tabular ):
                              'Superorder', 'Order', 'Suborder', 'Superfamily', 'Family', 'Subfamily',
                              'Tribe', 'Subtribe', 'Genus', 'Subgenus', 'Species', 'Subspecies'
                              ]
+
     def display_peek( self, dataset ):
         """Returns formated html of peek"""
         return Tabular.make_html_table( self, dataset, column_names=self.column_names )
@@ -384,6 +411,7 @@ class Sam( Tabular ):
         self.column_names = ['QNAME', 'FLAG', 'RNAME', 'POS', 'MAPQ', 'CIGAR',
                              'MRNM', 'MPOS', 'ISIZE', 'SEQ', 'QUAL', 'OPT'
                              ]
+
     def display_peek( self, dataset ):
         """Returns formated html of peek"""
         return Tabular.make_html_table( self, dataset, column_names=self.column_names )
@@ -423,7 +451,7 @@ class Sam( Tabular ):
                 line = fh.readline()
                 line = line.strip()
                 if not line:
-                    break #EOF
+                    break  # EOF
                 if line:
                     if line[0] != '@':
                         linePieces = line.split('\t')
@@ -447,7 +475,7 @@ class Sam( Tabular ):
             pass
         return False
 
-    def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 5, **kwd ):
+    def set_meta( self, dataset, overwrite=True, skip=None, max_data_lines=5, **kwd ):
         if dataset.has_data():
             dataset_fh = open( dataset.file_name )
             comment_lines = 0
@@ -487,9 +515,9 @@ class Sam( Tabular ):
             raise Exception('Result %s from %s' % (result, cmd))
     merge = staticmethod(merge)
 
-    # ------------- Dataproviders
+    # Dataproviders
     # sam does not use '#' to indicate comments/headers - we need to strip out those headers from the std. providers
-    #TODO:?? seems like there should be an easier way to do this - metadata.comment_char?
+    # TODO:?? seems like there should be an easier way to do this - metadata.comment_char?
     @dataproviders.decorators.dataprovider_factory( 'line', dataproviders.line.FilteredLineDataProvider.settings )
     def line_dataprovider( self, dataset, **settings ):
         settings[ 'comment_char' ] = '@'
@@ -546,10 +574,10 @@ class Sam( Tabular ):
         settings[ 'comment_char' ] = '@'
         return dataproviders.dataset.GenomicRegionDataProvider( dataset, 2, 3, 3, True, **settings )
 
-    #@dataproviders.decorators.dataprovider_factory( 'samtools' )
-    #def samtools_dataprovider( self, dataset, **settings ):
-    #    dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
-    #    return dataproviders.dataset.SamtoolsDataProvider( dataset_source, **settings )
+    # @dataproviders.decorators.dataprovider_factory( 'samtools' )
+    # def samtools_dataprovider( self, dataset, **settings ):
+    #     dataset_source = dataproviders.dataset.DatasetDataProvider( dataset )
+    #     return dataproviders.dataset.SamtoolsDataProvider( dataset_source, **settings )
 
 
 @dataproviders.decorators.has_dataproviders
@@ -571,7 +599,7 @@ class Pileup( Tabular ):
 
     def display_peek( self, dataset ):
         """Returns formated html of peek"""
-        return Tabular.make_html_table( self, dataset, column_parameter_alias={'chromCol':'Chrom', 'startCol':'Start', 'baseCol':'Base'} )
+        return Tabular.make_html_table( self, dataset, column_parameter_alias={'chromCol': 'Chrom', 'startCol': 'Start', 'baseCol': 'Base'} )
 
     def repair_methods( self, dataset ):
         """Return options for removing errors along with a description"""
@@ -612,7 +640,7 @@ class Pileup( Tabular ):
         except:
             return False
 
-    # ------------- Dataproviders
+    # Dataproviders
     @dataproviders.decorators.dataprovider_factory( 'genomic-region',
                                                     dataproviders.dataset.GenomicRegionDataProvider.settings )
     def genomic_region_dataprovider( self, dataset, **settings ):
@@ -636,7 +664,7 @@ class Vcf( Tabular ):
     column_names = [ 'Chrom', 'Pos', 'ID', 'Ref', 'Alt', 'Qual', 'Filter', 'Info', 'Format', 'data' ]
 
     MetadataElement( name="columns", default=10, desc="Number of columns", readonly=True, visible=False )
-    MetadataElement( name="column_types", default=['str','int','str','str','str','int','str','list','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False )
+    MetadataElement( name="column_types", default=['str', 'int', 'str', 'str', 'str', 'int', 'str', 'list', 'str', 'str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False )
     MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, optional=True, multiple=True, visible=False )
     MetadataElement( name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[] )
 
@@ -662,7 +690,7 @@ class Vcf( Tabular ):
             # Found header line, get sample names.
             dataset.metadata.sample_names = line.split()[ 9: ]
 
-    # ------------- Dataproviders
+    # Dataproviders
     @dataproviders.decorators.dataprovider_factory( 'genomic-region',
                                                     dataproviders.dataset.GenomicRegionDataProvider.settings )
     def genomic_region_dataprovider( self, dataset, **settings ):
@@ -685,6 +713,7 @@ class Eland( Tabular ):
     MetadataElement( name="reads", default=[], param=metadata.ListParameter, desc="Set of reads", readonly=True, visible=False, no_value=[] )
     MetadataElement( name="lanes", default=[], param=metadata.ListParameter, desc="Set of lanes", readonly=True, visible=False, no_value=[] )
     MetadataElement( name="barcodes", default=[], param=metadata.ListParameter, desc="Set of barcodes", readonly=True, visible=False, no_value=[] )
+
     def __init__(self, **kwd):
         """Initialize taxonomy datatype"""
         Tabular.__init__( self, **kwd )
@@ -693,6 +722,7 @@ class Eland( Tabular ):
                              'POSITION', 'STRAND', 'DESC', 'SRAS', 'PRAS', 'PART_CHROM'
                              'PART_CONTIG', 'PART_OFFSET', 'PART_STRAND', 'FILT'
                              ]
+
     def make_html_table( self, dataset, skipchars=None ):
         """Create HTML table, used for displaying peek"""
         if skipchars is None:
@@ -702,11 +732,11 @@ class Eland( Tabular ):
             # Generate column header
             out.append( '<tr>' )
             for i, name in enumerate( self.column_names ):
-                out.append( '<th>%s.%s</th>' % ( str( i+1 ), name ) )
+                out.append( '<th>%s.%s</th>' % ( str( i + 1 ), name ) )
             # This data type requires at least 11 columns in the data
             if dataset.metadata.columns - len( self.column_names ) > 0:
                 for i in range( len( self.column_names ), dataset.metadata.columns ):
-                    out.append( '<th>%s</th>' % str( i+1 ) )
+                    out.append( '<th>%s</th>' % str( i + 1 ) )
                 out.append( '</tr>' )
             out.append( self.make_html_peek_rows( dataset, skipchars=skipchars ) )
             out.append( '</table>' )
@@ -714,6 +744,7 @@ class Eland( Tabular ):
         except Exception, exc:
             out = "Can't create peek %s" % exc
         return out
+
     def sniff( self, filename ):
         """
         Determines whether the file is in ELAND export format
@@ -730,15 +761,15 @@ class Eland( Tabular ):
         try:
             compress = is_gzip(filename)
             if compress:
-               fh = gzip.GzipFile(filename, 'r')
+                fh = gzip.GzipFile(filename, 'r')
             else:
-               fh = open( filename )
+                fh = open( filename )
             count = 0
             while True:
                 line = fh.readline()
                 line = line.strip()
                 if not line:
-                    break #EOF
+                    break  # EOF
                 if line:
                     linePieces = line.split('\t')
                     if len(linePieces) != 22:
@@ -767,32 +798,32 @@ class Eland( Tabular ):
         fh.close()
         return False
 
-    def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 5, **kwd ):
+    def set_meta( self, dataset, overwrite=True, skip=None, max_data_lines=5, **kwd ):
         if dataset.has_data():
             compress = is_gzip(dataset.file_name)
             if compress:
-               dataset_fh = gzip.GzipFile(dataset.file_name, 'r')
+                dataset_fh = gzip.GzipFile(dataset.file_name, 'r')
             else:
-               dataset_fh = open( dataset.file_name )
+                dataset_fh = open( dataset.file_name )
             lanes = {}
             tiles = {}
             barcodes = {}
             reads = {}
-            #    # Should always read the entire file (until we devise a more clever way to pass metadata on)
-            #if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize:
-            #    # If the dataset is larger than optional_metadata, just count comment lines.
-            #    dataset.metadata.data_lines = None
-            #else:
-            #    # Otherwise, read the whole thing and set num data lines.
+            #     # Should always read the entire file (until we devise a more clever way to pass metadata on)
+            # if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize:
+            #     # If the dataset is larger than optional_metadata, just count comment lines.
+            #     dataset.metadata.data_lines = None
+            # else:
+            #     # Otherwise, read the whole thing and set num data lines.
             for i, line in enumerate(dataset_fh):
                 if line:
                     linePieces = line.split('\t')
                     if len(linePieces) != 22:
-                        raise Exception('%s:%d:Corrupt line!' % (dataset.file_name,i))
-                    lanes[linePieces[2]]=1
-                    tiles[linePieces[3]]=1
-                    barcodes[linePieces[6]]=1
-                    reads[linePieces[7]]=1
+                        raise Exception('%s:%d:Corrupt line!' % (dataset.file_name, i))
+                    lanes[linePieces[2]] = 1
+                    tiles[linePieces[3]] = 1
+                    barcodes[linePieces[6]] = 1
+                    reads[linePieces[7]] = 1
                 pass
             dataset.metadata.data_lines = i + 1
             dataset_fh.close()
@@ -816,7 +847,81 @@ class FeatureLocationIndex( Tabular ):
     """
     An index that stores feature locations in tabular format.
     """
-    file_ext='fli'
+    file_ext = 'fli'
     MetadataElement( name="columns", default=2, desc="Number of columns", readonly=True, visible=False )
     MetadataElement( name="column_types", default=['str', 'str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False, no_value=[] )
 
+
+@dataproviders.decorators.has_dataproviders
+class CSV( TabularData ):
+    """
+    Delimiter-separated table data.
+    This includes CSV, TSV and other dialects understood by the
+    Python 'csv' module https://docs.python.org/2/library/csv.html
+    """
+    delimiter = ','
+    file_ext = 'csv'  # File extension
+    peek_size = 1024  # File chunk used for sniffing CSV dialect
+
+    def is_int( self, column_text ):
+        try:
+            int( column_text )
+            return True
+        except:
+            return False
+
+    def is_float( self, column_text ):
+        try:
+            float( column_text )
+            return True
+        except:
+            if column_text.strip().lower() == 'na':
+                return True  # na is special cased to be a float
+            return False
+
+    def guess_type( self, text ):
+        if self.is_int(text):
+            return 'int'
+        if self.is_float(text):
+            return 'float'
+        else:
+            return 'str'
+
+    def sniff( self, filename ):
+        """ Return True if if recognizes dialect and header. """
+        if not csv.Sniffer().has_header(open(filename, 'r').read(self.peek_size)):
+            return False
+        # Fetch at least three consecutive lines to be reasonably sure
+        reader = csv.reader(open(filename, 'r'))
+        for i in range(0, 3):
+            reader.next()
+        return True
+
+    def set_meta( self, dataset, **kwd ):
+        with open(dataset.file_name, 'r') as csvfile:
+            # Parse file
+            reader = csv.reader(csvfile)
+            data_row = None
+            header_row = None
+            try:
+                header_row = reader.next()
+                data_row = reader.next()
+                for row in reader:
+                    pass
+            except csv.Error as e:
+                raise Exception('CSV reader error - line %d: %s' % (reader.line_num, e))
+
+            # Guess column types
+            if len(header_row) != len(data_row):
+                raise ('mismatching number of columns in header and data')
+            column_types = []
+            for cell in data_row:
+                column_types.append(self.guess_type(cell))
+
+            # Set metadata
+            dataset.metadata.data_lines = reader.line_num - 1
+            dataset.metadata.comment_lines = 1
+            dataset.metadata.column_types = column_types
+            dataset.metadata.columns = len(header_row)
+            dataset.metadata.column_names = header_row
+            dataset.metadata.delimiter = reader.dialect.delimiter

lib/galaxy/datatypes/text.py

@@ -48,12 +48,12 @@ class Json( Text ):
         else:
             with open(filename, "r") as fh:
                 while True:
-                    line = fh.readline()
-                    line = line.strip()
-                    if line:
+                    # Grab first chunk of file and see if it looks like json.
+                    start = fh.read(100).strip()
+                    if start:
                         # simple types are valid JSON as well - but would such a file
                         # be interesting as JSON in Galaxy?
-                        return line.startswith("[") or line.startswith("{")
+                        return start.startswith("[") or start.startswith("{")
             return False
 
     def display_peek( self, dataset ):

lib/galaxy/visualization/plugins/resource_parser.py

@@ -23,7 +23,7 @@ class ResourceParser( object ):
     in a new dictionary.
 
     The keys used to store the new values can optionally be re-mapped to
-    new keys (e.g. dataset_id="NNN" -> hda=<HistoryDatasetAsscoation>).
+    new keys (e.g. dataset_id="NNN" -> hda=<HistoryDatasetAssociation>).
     """
     primitive_parsers = {
         'str'   : lambda param: galaxy.util.sanitize_html.sanitize_html( param, 'utf-8' ),

lib/galaxy/web/base/controllers/admin.py

@@ -32,10 +32,10 @@ class Admin( object ):
         message = escape( kwd.get( 'message', ''  ) )
         status = kwd.get( 'status', 'done' )
         if trans.webapp.name == 'galaxy':
-            installed_repositories = trans.install_model.context.query( trans.install_model.ToolShedRepository ).first()
+            is_repo_installed = trans.install_model.context.query( trans.install_model.ToolShedRepository ).first() is not None
             installing_repository_ids = suc.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=True )
             return trans.fill_template( '/webapps/galaxy/admin/index.mako',
-                                        installed_repositories=installed_repositories,
+                                        is_repo_installed=is_repo_installed,
                                         installing_repository_ids=installing_repository_ids,
                                         message=message,
                                         status=status )
@@ -50,7 +50,11 @@ class Admin( object ):
         message = escape( kwd.get( 'message', ''  ) )
         status = kwd.get( 'status', 'done' )
         if trans.webapp.name == 'galaxy':
+            is_repo_installed = trans.install_model.context.query( trans.install_model.ToolShedRepository ).first() is not None
+            installing_repository_ids = suc.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=True )
             return trans.fill_template( '/webapps/galaxy/admin/center.mako',
+                                        is_repo_installed=is_repo_installed,
+                                        installing_repository_ids=installing_repository_ids,
                                         message=message,
                                         status=status )
         else:

lib/galaxy/web/base/interactive_environments.py

@@ -8,6 +8,7 @@ import subprocess
 from galaxy.util.bunch import Bunch
 from galaxy import web
 from galaxy.managers import api_keys
+from galaxy.tools.deps.docker_util import DockerVolume
 
 import logging
 log = logging.getLogger(__name__)
@@ -154,7 +155,10 @@ class InteractiveEnviornmentRequest(object):
             .replace('${PORT}', str(self.attr.PORT))
         return url
 
-    def docker_cmd(self, env_override={}):
+    def volume(self, host_path, container_path, **kwds):
+        return DockerVolume(host_path, container_path, **kwds)
+
+    def docker_cmd(self, env_override={}, volumes=[]):
         """
             Generate and return the docker command to execute
         """
@@ -162,16 +166,19 @@ class InteractiveEnviornmentRequest(object):
         conf = self.get_conf_dict()
         conf.update(env_override)
         env_str = ' '.join(['-e "%s=%s"' % (key.upper(), item) for key, item in conf.items()])
-        return '%s run %s -d %s -p %s:%s -v "%s:/import/" %s' % \
+        volume_str = ' '.join(['-v "%s"' % volume for volume in volumes])
+        return '%s run %s -d %s -p %s:%s -v "%s:/import/" %s %s' % \
             (self.attr.viz_config.get("docker", "command"),
             env_str,
             self.attr.viz_config.get("docker", "command_inject"),
             self.attr.PORT, self.attr.docker_port,
-            temp_dir, self.attr.viz_config.get("docker", "image"))
+            temp_dir,
+            volume_str,
+            self.attr.viz_config.get("docker", "image"))
 
-    def launch(self, raw_cmd=None, env_override={}):
+    def launch(self, raw_cmd=None, env_override={}, volumes=[]):
         if raw_cmd is None:
-            raw_cmd = self.docker_cmd(env_override=env_override)
+            raw_cmd = self.docker_cmd(env_override=env_override, volumes=volumes)
         log.info("Starting docker container for IE {0} with command [{1}]".format(
             self.attr.viz_id,
             raw_cmd

lib/tool_shed/metadata/metadata_generator.py

@@ -630,10 +630,10 @@ class MetadataGenerator( object ):
         tool_requirements = []
         for tool_requirement in tool.requirements:
             name = str( tool_requirement.name )
-            type = str( tool_requirement.type )
+            tool_type = str( tool_requirement.type )
             version = str( tool_requirement.version ) if tool_requirement.version else None
             requirement_dict = dict( name=name,
-                                     type=type,
+                                     type=tool_type,
                                      version=version )
             tool_requirements.append( requirement_dict )
         # Handle tool.tests.
@@ -665,9 +665,13 @@ class MetadataGenerator( object ):
                                 collection_type = values.collection_type
                                 metadata_display_value = "%s collection" % collection_type
                                 inputs.append( ( param_name, metadata_display_value ) )
-                            elif len( values ) == 1:
-                                inputs.append( ( param_name, values[ 0 ] ) )
                             else:
+                                try:
+                                    if len( values ) == 1:
+                                        inputs.append( ( param_name, values[ 0 ] ) )
+                                        continue
+                                except TypeError:
+                                    log.exception( 'Expected a list of values for tool "%s" parameter "%s", got %s: %s', tool.id, param_name, type( values ), values )
                                 inputs.append( ( param_name, values ) )
                 outputs = []
                 for output in ttb.outputs:

static/welcome.rst

@@ -25,11 +25,11 @@ Need Help?
 
 * `Galaxy documentation`__
 * `Galaxy wiki`__
-* Report bugs and problems to Galaxy mailing list: `galaxy-dev@bx.psu.edu`__
+* Report bugs and problems to Galaxy mailing list: `galaxy-dev@lists.galaxyproject.org`__
 
 .. __: /static/help.html
 .. __: http://wiki.galaxyproject.org/
-.. __: mailto:galaxy-dev@bx.psu.edu
+.. __: mailto:galaxy-dev@lists.galaxyproject.org
 
 Version: <b>2.1</b> Revision: <b>$Rev$</b>
 

templates/export_base.mako

@@ -6,10 +6,12 @@
     def inherit(context):
         if context.get('use_panels', False) == True:
             if context.get('webapp'):
-                webapp = context.get('webapp')
+                app_name = context.get('webapp')
+            elif context.get('app'):
+                app_name = context.get('app').name
             else:
-                webapp = 'galaxy'
-            return '/webapps/%s/base_panels.mako' % webapp
+                app_name = 'galaxy'
+            return '/webapps/%s/base_panels.mako' % app_name
         else:
             return '/base.mako'
 %>

templates/form.mako

@@ -2,10 +2,12 @@
     def inherit(context):
         if context.get('use_panels'):
             if context.get('webapp'):
-                webapp = context.get('webapp')
+                app_name = context.get('webapp')
+            elif context.get('app'):
+                app_name = context.get('app').name
             else:
-                webapp = 'galaxy'
-            return '/webapps/%s/base_panels.mako' % webapp
+                app_name = 'galaxy'
+            return '/webapps/%s/base_panels.mako' % app_name
         else:
             return '/base.mako'
 %>

templates/grid_base.mako

@@ -9,10 +9,12 @@
 
         if context.get('use_panels'):
             if context.get('webapp'):
-                webapp = context.get('webapp')
+                app_name = context.get('webapp')
+            elif context.get('app'):
+                app_name = context.get('app').name
             else:
-                webapp = 'galaxy'
-            return '/webapps/%s/base_panels.mako' % webapp
+                app_name = 'galaxy'
+            return '/webapps/%s/base_panels.mako' % app_name
         else:
             return '/base.mako'
 %>

templates/ind_share_base.mako

@@ -6,10 +6,12 @@
     def inherit(context):
         if context.get('use_panels'):
             if context.get('webapp'):
-                webapp = context.get('webapp')
+                app_name = context.get('webapp')
+            elif context.get('app'):
+                app_name = context.get('app').name
             else:
-                webapp = 'galaxy'
-            return '/webapps/%s/base_panels.mako' % webapp
+                app_name = 'galaxy'
+            return '/webapps/%s/base_panels.mako' % app_name
         else:
             return '/base.mako'
 %>

templates/message.mako

@@ -2,10 +2,12 @@
     def inherit(context):
         if context.get('use_panels'):
             if context.get('webapp'):
-                webapp = context.get('webapp')
+                app_name = context.get('webapp')
+            elif context.get('app'):
+                app_name = context.get('app').name
             else:
-                webapp = 'galaxy'
-            return '/webapps/%s/base_panels.mako' % webapp
+                app_name = 'galaxy'
+            return '/webapps/%s/base_panels.mako' % app_name
         else:
             return '/base.mako'
 %>

templates/sharing_base.mako

@@ -6,10 +6,12 @@
     def inherit(context):
         if context.get('use_panels', False) == True:
             if context.get('webapp'):
-                webapp = context.get('webapp')
+                app_name = context.get('webapp')
+            elif context.get('app'):
+                app_name = context.get('app').name
             else:
-                webapp = 'galaxy'
-            return '/webapps/%s/base_panels.mako' % webapp
+                app_name = 'galaxy'
+            return '/webapps/%s/base_panels.mako' % app_name
         else:
             return '/base.mako'
 %>

templates/webapps/galaxy/admin/center.mako

@@ -4,189 +4,107 @@
 <%def name="title()">Galaxy Administration</%def>
 
 <h2>Administration</h2>
-
+Please visit <a href="https://wiki.galaxyproject.org/Admin" target="_blank">the Galaxy administration hub</a> to learn how to keep your Galaxy in best shape.
 %if message:
     ${render_msg( message, status )}
 %else:
-    <p>The menu on the left provides the following features</p>
-    <ul>
-        <li><strong>Security</strong> - see the <strong>Data Security and Data Libraries</strong> section below for details
-            <p/>
+        <h4>Server</h4>
             <ul>
                 <li>
-                    <strong>Manage users</strong> - provides a view of the registered users and all groups and non-private roles associated 
-                    with each user.  
+                    <strong>Data types registry</strong> - See all datatypes available in this Galaxy.
                 </li>
-                <p/>
                 <li>
-                    <strong>Manage groups</strong> - provides a view of all groups along with the members of the group and the roles associated with
-                    each group (both private and non-private roles).  The group names include a link to a page that allows you to manage the users and 
-                    roles that are associated with the group.
+                    <strong>Data tables registry</strong> - See all data tables available in this Galaxy.
                 </li>
-                <p/>
                 <li>
-                    <strong>Manage roles</strong> - provides a view of all non-private roles along with the role type, and the users and groups that
-                    are associated with the role.  The role names include a link to a page that allows you to manage the users and groups that are associated 
-                    with the role.  The page also includes a view of the data library datasets that are associated with the role and the permissions applied 
-                    to each dataset.
+                    <strong>Display applications</strong> - See all display applications configured in this Galaxy.
+                </li>
+                <li>
+                    <strong>Manage jobs</strong> - Display all jobs that are currently not finished (i.e., their state is new, waiting, queued, or running).  Administrators are able to cleanly stop long-running jobs. 
                 </li>
             </ul>
-        </li>
-        <p/>
-        <li><strong>Data</strong>
-            <p/>
+
+        <h4>Tools and Tool Shed</h4>
+            <ul>
+            %if trans.app.tool_shed_registry and trans.app.tool_shed_registry.tool_sheds:
+                <li>
+                    <strong>Search Tool Shed</strong> - Search and install new tools and other Galaxy utilities from the Tool Shed. See <a href="https://wiki.galaxyproject.org/Admin/Tools/AddToolFromToolShedTutorial" target="_blank">the tutorial</a>.
+                </li>
+            %endif
+            %if tool_shed_repository_ids:
+                <li>
+                    <strong>Monitor installing repositories</strong> - View the status of tools that are being currently installed.
+                </li>
+            %endif
+            %if is_repo_installed:
+                <li>
+                    <strong>Manage installed tools</strong> - View and administer installed tools and utilities on this Galaxy.
+                </li>
+                <li>
+                    <strong>Reset metadata</strong> - Select on which repositories you want to reset metadata.
+                </li>
+            %endif
+                <li>
+                    <strong>Download local tool</strong> - Download a tarball with a tool from this Galaxy.
+                </li>
+                <li>
+                    <strong>Reload a tool's configuration</strong> - Allows a new version of a tool to be loaded while the server is running.
+                </li>
+                <li>
+                    <strong>Tool lineage</strong> - A view of a version lineages for all installed tools. Useful for debugging.
+                </li>
+                <li>
+                    <strong>Review tool migration stages</strong> - See the list of migration stages that moved sets of tools from the distribution to the Tool Shed.
+                </li>
+            </ul>
+
+        <h4>Security</h4>
             <ul>
                 <li>
-                    <strong>Manage data libraries</strong> - Data libraries enable a Galaxy administrator to upload datasets into a data library.  Currently,
-                    only administrators can create data libraries.
-                    <p/>
-                    When a data library is first created, it is considered "public" since it will be displayed in the "Data Libraries" view for any user, even 
-                    those that are not logged in.  The Galaxy administrator can restrict access to a data library by associating roles with the data library's 
-                    "access library" permission.  This permission will conservatively override the [dataset] "access" permission for the data library's contained 
-                    datasets.
-                    <p/>
-                    For example, if a data library's "access library" permission is associated with Role1 and the data library contains "public" datasets, the 
-                    data library will still only be displayed to those users that have Role1.  However, if the data library's "access library" permission is 
-                    associated with both Role1 and Role2 and the data library contains datasets whose [dataset] "access" permission is associated with only Role1, 
-                    then users that have Role2 will be able to access the library, but will not see those contained datasets whose [dataset] "access" permission 
-                    is associated with only Role1.
-                    <p/>
-                    In addition to the "access library" permission, permission to perform the following functions on the data library (and its contents) can 
-                    be granted to users (a library item is one of: a data library, a library folder, a library dataset).
-                    <p/>
-                    <ul>
-                        <li><strong>add library item</strong> - Users that have the role can add library items to this data library or folder</li>
-                        <li><strong>modify library item</strong> - Users that have the role can modify this library item</li>
-                        <li><strong>manage library permissions</strong> - Users that have the role can manage permissions applied to this library item</li>
-                    </ul>
-                    <p/>
-                    The default behavior is for no permissions to be applied to a data library item, but applied permissions are inherited downward (with the exception
-                    of the "access library" permission, which is only available on the data library itself).  Because of this, it is important to set desired permissions 
-                    on a new data library when it is created.  When this is done, new folders and datasets added to the data library will automatically inherit those 
-                    permissions.  In the same way, permissions can be applied to a folder, which will be automatically inherited by all contained datasets and sub-folders.
-                    <p/>
-                    The "Data Libraries" menu item allows users to access the datasets in a data library as long as they are not restricted from accessing them.
-                    Importing a library dataset into a history will not make a copy of the dataset, but will be a "pointer" to the dataset on disk.  This
-                    approach allows for multiple users to use a single (possibly very large) dataset file.
+                    <strong>Users</strong> - A view of all users and all groups and non-private roles associated with each user.  
+                </li>
+                <li>
+                    <strong>Groups</strong> - A view of all groups along with the members of the group and the roles associated with each group.
+                </li>
+                <li>
+                    <strong>Roles</strong> - A view of all non-private roles along with the role type, and the users and groups that are associated with the role.
+                    Also includes a view of the data library datasets that are associated with the role and the permissions applied to each dataset.
+                </li>
+                <li>
+                    <strong>API keys</strong> - A view of all generated API keys with an option to re-generate.
+                </li>
+            %if trans.app.config.allow_user_impersonation:
+                <li>
+                    <strong>Impersonate a user</strong> - Allows to view Galaxy as another user in order to help troubleshoot issues.
+                </li>
+            %endif
+            </ul>
+
+        <h4>Data</h4>
+            <ul>
+            %if trans.app.config.enable_quotas:
+                <li>
+                    <strong>Quotas</strong> - Manage user space quotas. See <a href="https://wiki.galaxyproject.org/Admin/DiskQuotas" target="_blank">wiki</a> for details.
+                </li>
+            %endif
+                <li>
+                    <strong>Data libraries</strong> - Data libraries enable authorized Galaxy users to share datasets with other groups or users. Only administrators can create data libraries. See <a href="https://wiki.galaxyproject.org/DataLibraries" target="_blank">wiki</a> for details and <a href="https://wiki.galaxyproject.org/Admin/DataLibraries/LibrarySecurity" target="_blank">this page</a> for security description.
+                </li>
+                <li>
+                    <strong>Local data</strong> - Manage the reference (and other) data that is stored within Tool Data Tables. See <a href="https://wiki.galaxyproject.org/Admin/Tools/DataManagers" target="_blank">wiki</a> for details.
                 </li>
             </ul>
-        </li>
-        <p/>
-        <li><strong>Server</strong>
-            <p/>
+        <h4>Form definitions</h4>
             <ul>
                 <li>
-                    <strong>Reload a tool's configuration</strong> - allows a new version of a tool to be loaded while the server is running
-                </li>
-                <p/>
-                <li>
-                    <strong>Profile memory usage</strong> - measures system memory used for certain Galaxy functions
-                </li>
-                <p/>
-                <li>
-                    <strong>Manage jobs</strong> - displays all jobs that are currently not finished (i.e., their state is new, waiting, queued, or
-                    running).  Administrators are able to cleanly stop long-running jobs. 
+                    <strong>Form definitions</strong> - Manage local form definitions.
                 </li>
             </ul>
-        </li>
-        <p/>
-        <li><strong>Forms</strong>
-            <p/>To be completed
-        </li>
-        <p/>
-        <li><strong>Sequencing Requests</strong>
-            <p/>To be completed
-        </li>
-        <p/>
-        <li><strong>Cloud</strong>
-            <p/>To be completed
-        </li>
-    </ul>
-    <p/>
-    <p><strong>Data Security and Data Libraries</strong></p>
-    <p/>
-    <strong>Security</strong> - Data security in Galaxy is a new feature, so familiarize yourself with the details which can be found 
-    here or in our <a href="http://wiki.galaxyproject.org/Learn/Security%20Features" target="_blank">data security page</a>.  The data security 
-    process incorporates users, groups and roles, and enables the application of certain permissions on datasets, specifically "access"
-    and "manage permissions".  By default, the "manage permissions" permission is associated with the dataset owner's private role, and
-    the "access" permission is not set, making the dataset public.  With these default permissions, users should not see any difference
-    in the way Galaxy has behaved in the past.
-    <ul>
-        <li>
-            <strong>Users</strong> - registered Galaxy users that have created a Galaxy account.  Users can belong to groups and can
-            be associated with 1 or more roles.  If a user is not authenticated during a Galaxy session, they will not have access to any 
-            of the security features, and datasets they create during that session will have no permissions applied to them (i.e., they
-            will be considered "public", and no one will be allowed to change permissions on them).
-        </li>
-        <p/>
-        <li>
-            <strong>Groups</strong> - a set of 0 or more users which are considered members of the group.  Groups can be associated with 0
-            or more roles, simplifying the process of applying permissions to the data between a select group of users.
-        </li>
-        <p/>
-        <li>
-            <strong>Roles</strong> - associate users and groups with specific permissions on datasets.  For example, users in groups A and B 
-            can be associated with role C which gives them the "access" permission on datasets D, E and F.  Roles have a type which is currently
-            one of the following:
+
+        <h4>Sample tracking</h4>
             <ul>
                 <li>
-                    <strong>private</strong> - every user is associated automatically with their own private role.  Administrators cannot 
-                    manage private roles.
+                    Please see the <a href="https://wiki.galaxyproject.org/Admin/DataLibraries/LibrarySampleTracking" target="_blank">sample tracking tutorial</a>.
                 </li>
-                <li>
-                    <strong>user</strong> - this is currently not used, but eventually any registered user will be able to create a new role
-                    and this will be its type.
-                </li>
-                <li>
-                    <strong>sharing</strong> - a role created automatically during a Galaxy session that enables a user to share data with 
-                    another user.  This can generally be considered a temporary role.
-                </li>
-                <li><strong>admin</strong> - a role created by a Galaxy administrator.</li>
             </ul>
-        </li>
-        <p/>
-        <li>
-            <strong>Dataset Permissions</strong> - applying the following permissions will to a dataset will result in the behavior described.
-            <ul>
-                <li>
-                    <strong>access</strong> - users associated with the role can import this dataset into their history for analysis.
-                    <p>
-                        If no roles with the "access" permission are associated with a dataset, the dataset is "public" and may be accessed by anyone 
-                        that can access the data library in which it is contained.  See the <strong>Manage data libraries</strong> section above for 
-                        details.  Public datasets contained in public data libraries will be accessible to all users (as well as anyone not logged in 
-                        during a Galaxy session) from the list of data libraries displayed when the "Data Libraries" menu item is selected.
-                    </p>
-                    <p>
-                        Associating a dataset with a role that includes the "access" permission restricts the set of users that can access it.  
-                        For example, if 'Role A' includes the "access" permission and 'Role A' is associated with the dataset, only those users
-                        and groups who are associated with 'Role A' may access the dataset.
-                    </p>
-                    <p>
-                        If multiple roles that include the "access" permission are associated with a dataset, access to the dataset is derived
-                        from the intersection of the users associated with the roles.  For example, if 'Role A' and 'Role B' are associated with
-                        a dataset, only those users and groups who are associated with both 'Role A' AND 'Role B' may access the dataset.  When
-                        the "access" permission is applied to a dataset, Galaxy checks to make sure that at least 1 user belongs to all groups and
-                        roles associated with the "access" permission (otherwise the dataset would be restricted from everyone).
-                    </p>
-                    <p>
-                        In order for a user to make a dataset private (i.e., only they can access it), they should associate the dataset with
-                        their private role (the role identical to their Galaxy user name / email address).  Associating additional roles that
-                        include the "access" permission is not possible, since it would render the dataset inaccessible to everyone.
-                    <p>
-                        To make a dataset private to themselves and one or more other users, the user can create a new role and associate the dataset 
-                        with that role, not their "private role".  Galaxy makes this easy by telling the user they are about to share a private dataset 
-                        and giving them the option of doing so.  If they respond positively, the sharing role is automatically created for them.
-                    </p>
-                    <p>
-                        Private data (data associated with roles that include the "access" permission) must be made public in order to be used
-                        with external applications like the "view at UCSC" link, or the "Perform genome analysis and prediction with EpiGRAPH" 
-                        tool.  Being made publically accessible means removing the association of all roles that include the "access" permission
-                        from the dataset.
-                    <p>
-                </li>
-                <li><strong>manage permissions</strong> - Role members can manage the permissions applied to this dataset</li>
-            </ul>
-        </li>
-    </ul>
-    <br/>
 %endif

templates/webapps/galaxy/admin/index.mako

@@ -46,13 +46,43 @@
     <div class="unified-panel-body">
         <div class="toolMenu">
             <div class="toolSectionList">
+                <div class="toolSectionTitle">Server</div>
+                <div class="toolSectionBody">
+                    <div class="toolSectionBg">
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='view_datatypes_registry' )}" target="galaxy_main">Data types registry</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='view_tool_data_tables' )}" target="galaxy_main">Data tables registry</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='display_applications' )}" target="galaxy_main">Display applications</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='jobs' )}" target="galaxy_main">Manage jobs</a></div>
+                    </div>
+                </div>
+                <div class="toolSectionPad"></div>
+                <div class="toolSectionTitle">Tools and Tool Shed</div>
+                <div class="toolSectionBody">
+                    <div class="toolSectionBg">                        
+                    %if trans.app.tool_shed_registry and trans.app.tool_shed_registry.tool_sheds:
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='browse_tool_sheds' )}" target="galaxy_main">Search Tool Shed</a></div>
+                    %endif
+                    %if installing_repository_ids:
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='monitor_repository_installation', tool_shed_repository_ids=installing_repository_ids )}" target="galaxy_main">Monitor installing repositories</a></div>
+                    %endif
+                    %if is_repo_installed:
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='browse_repositories' )}" target="galaxy_main">Manage installed tools</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='reset_metadata_on_selected_installed_repositories' )}" target="galaxy_main">Reset metadata</a></div>
+                    %endif
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='package_tool' )}" target="galaxy_main">Download local tool</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='tool_versions' )}" target="galaxy_main">Tool lineage</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='reload_tool' )}" target="galaxy_main">Reload a tool's configuration</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='review_tool_migration_stages' )}" target="galaxy_main">Review tool migration stages</a></div>
+                    </div>
+                </div>
+                <div class="toolSectionPad"></div>
                 <div class="toolSectionTitle">Security</div>
                 <div class="toolSectionBody">
                     <div class="toolSectionBg">
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='users' )}" target="galaxy_main">Manage users</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='groups' )}" target="galaxy_main">Manage groups</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='roles' )}" target="galaxy_main">Manage roles</a></div>
-			            <div class="toolTitle"><a href="${h.url_for( controller='userskeys', action='all_users' )}" target="galaxy_main">Manage users API keys</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='users' )}" target="galaxy_main">Users</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='groups' )}" target="galaxy_main">Groups</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='roles' )}" target="galaxy_main">Roles</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='userskeys', action='all_users' )}" target="galaxy_main">API keys</a></div>
                         %if trans.app.config.allow_user_impersonation:
                             <div class="toolTitle"><a href="${h.url_for( controller='admin', action='impersonate' )}" target="galaxy_main">Impersonate a user</a></div>
                         %endif
@@ -63,55 +93,25 @@
                 <div class="toolSectionBody">
                     <div class="toolSectionBg">
                         %if trans.app.config.enable_quotas:
-                            <div class="toolTitle"><a href="${h.url_for( controller='admin', action='quotas' )}" target="galaxy_main">Manage quotas</a></div>
+                            <div class="toolTitle"><a href="${h.url_for( controller='admin', action='quotas' )}" target="galaxy_main">Quotas</a></div>
                         %endif
-                        <div class="toolTitle"><a href="${h.url_for( controller='library_admin', action='browse_libraries' )}" target="galaxy_main">Manage data libraries</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='data_manager' )}" target="galaxy_main">Manage local data (beta)</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='library_admin', action='browse_libraries' )}" target="galaxy_main">Data libraries</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='data_manager' )}" target="galaxy_main">Local data</a></div>
                     </div>
                 </div>
                 <div class="toolSectionPad"></div>
-                <div class="toolSectionTitle">Server</div>
-                <div class="toolSectionBody">
-                    <div class="toolSectionBg">
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='view_datatypes_registry' )}" target="galaxy_main">View data types registry</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='view_tool_data_tables' )}" target="galaxy_main">View data tables registry</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='display_applications' )}" target="galaxy_main">View display applications</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='tool_versions' )}" target="galaxy_main">View tool lineage</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='package_tool' )}" target="galaxy_main">Download tool tarball</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='reload_tool' )}" target="galaxy_main">Reload a tool's configuration</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='jobs' )}" target="galaxy_main">Manage jobs</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='admin', action='review_tool_migration_stages' )}" target="galaxy_main">Review tool migration stages</a></div>
-                        %if installing_repository_ids:
-                            <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='monitor_repository_installation', tool_shed_repository_ids=installing_repository_ids )}" target="galaxy_main">Monitor installing tool shed repositories</a></div>
-                        %endif
-                        %if installed_repositories:
-                            <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='reset_metadata_on_selected_installed_repositories' )}" target="galaxy_main">Reset metadata for tool shed repositories</a></div>
-                            <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='browse_repositories' )}" target="galaxy_main">Manage installed tool shed repositories</a></div>
-                        %endif
-                    </div>
-                </div>
-                %if trans.app.tool_shed_registry and trans.app.tool_shed_registry.tool_sheds:
-                    <div class="toolSectionPad"></div>
-                    <div class="toolSectionTitle">Tool sheds</div>
-                    <div class="toolSectionBody">
-                        <div class="toolSectionBg">                        
-                            <div class="toolTitle"><a href="${h.url_for( controller='admin_toolshed', action='browse_tool_sheds' )}" target="galaxy_main">Search and browse tool sheds</a></div>
-                        </div>
-                    </div>
-                %endif
-                <div class="toolSectionPad"></div>
                 <div class="toolSectionTitle">Form Definitions</div>
                 <div class="toolSectionBody">
                     <div class="toolSectionBg">
-                        <div class="toolTitle"><a href="${h.url_for( controller='forms', action='browse_form_definitions' )}" target="galaxy_main">Manage form definitions</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='forms', action='browse_form_definitions' )}" target="galaxy_main">Form definitions</a></div>
                     </div>
                 </div>
                 <div class="toolSectionPad"></div>
                 <div class="toolSectionTitle">Sample Tracking</div>
                 <div class="toolSectionBody">
                     <div class="toolSectionBg">
-                        <div class="toolTitle"><a href="${h.url_for( controller='external_service', action='browse_external_services' )}" target="galaxy_main">Manage sequencers and external services</a></div>
-                        <div class="toolTitle"><a href="${h.url_for( controller='request_type', action='browse_request_types' )}" target="galaxy_main">Manage request types</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='external_service', action='browse_external_services' )}" target="galaxy_main">Sequencers and external services</a></div>
+                        <div class="toolTitle"><a href="${h.url_for( controller='request_type', action='browse_request_types' )}" target="galaxy_main">Request types</a></div>
                         <div class="toolTitle"><a href="${h.url_for( controller='requests_admin', action='browse_requests' )}" target="galaxy_main">Sequencing requests</a></div>
                         <div class="toolTitle"><a href="${h.url_for( controller='requests_common', action='find_samples', cntrller='requests_admin' )}" target="galaxy_main">Find samples</a></div>
                     </div>

tools/data_source/upload.xml

@@ -213,6 +213,12 @@ Any data in tab delimited format (tabular)
 
 -----
 
+**Table (delimiter-separated)**
+
+Any delimiter-separated tabular data (CSV or TSV).
+
+-----
+
 **Wig**
 
 The wiggle format is line-oriented.  Wiggle data is preceded by a track definition line, which adds a number of options for controlling the default display of this track.

tools/stats/gsummary.py

@@ -1,7 +1,12 @@
 #!/usr/bin/env python
 
 import sys, re, tempfile
-from rpy import *
+try:
+    from rpy2.rpy_classic import *
+except:
+    # RPy isn't maintained, and doesn't work with R>3.0, use it as a fallback
+    from rpy import *
+
 # Older py compatibility
 try:
     set()
@@ -87,11 +92,11 @@ def main():
         stop_err( "Invalid column or column data values invalid for computation.  See tool tips and syntax for data requirements." )
     else:
         # summary function and return labels
+        set_default_mode( NO_CONVERSION )
         summary_func = r( "function( x ) { c( sum=sum( as.numeric( x ), na.rm=T ), mean=mean( as.numeric( x ), na.rm=T ), stdev=sd( as.numeric( x ), na.rm=T ), quantile( as.numeric( x ), na.rm=TRUE ) ) }" )
         headings = [ 'sum', 'mean', 'stdev', '0%', '25%', '50%', '75%', '100%' ]
         headings_str = "\t".join( headings )
         
-        set_default_mode( NO_CONVERSION )
         r_data_frame = r.read_table( tmp_file.name, header=True, sep="\t" )
         
         outfile = open( outfile_name, 'w' )
@@ -105,7 +110,7 @@ def main():
             stop_err( "Computation resulted in the following error: %s" % str( s ) )
         summary = summary.as_py( BASIC_CONVERSION )
         outfile.write( "#%s\n" % headings_str )
-        outfile.write( "%s\n" % "\t".join( [ "%g" % ( summary[ k ] ) for k in headings ] ) )
+        outfile.write( "%s\n" % "\t".join( [ "%g" % k for k in summary ] ) )
         outfile.close()
 
         if skipped_lines: