diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f1503dad1d..e78f10f3715 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,13 +50,12 @@ the [development mailing list](http://dev.list.galaxyproject.org/). * Most bug fixes to previously release components (things in galaxy-dist) should be made against the recent `release_XX.XX` branch (`git checkout release_XX.XX`). - * Serious security problems should not be fixed via pull request - - please responsibly disclose these by e-mailing them (with or - without patches) to galaxy-lab@bx.psu.edu. The Galaxy core - development team will issue patches to public servers before - announcing the issue to ensure there is time to patch and - highlight these fixes widely. We will provide you credit for the - discovery when publicly disclosing the issue. + * Serious security problems should not be fixed via pull request - please + responsibly disclose these by e-mailing them (with or without patches) to + galaxy-lab@lists.galaxyproject.org. The Galaxy core development team will + issue patches to public servers before announcing the issue to ensure there + is time to patch and highlight these fixes widely. We will provide you + credit for the discovery when publicly disclosing the issue. * If your changes modify code - please ensure the resulting files conform to Galaxy [style diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 6582e3b0d05..e24226f0be6 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -51,7 +51,7 @@ - + @@ -315,6 +315,7 @@ + diff --git a/config/plugins/visualizations/charts/config/charts.xml b/config/plugins/visualizations/charts/config/charts.xml index b3624fa1bf4..8356fc32d5f 100644 --- a/config/plugins/visualizations/charts/config/charts.xml +++ b/config/plugins/visualizations/charts/config/charts.xml @@ -5,6 +5,7 @@ HistoryDatasetAssociation tabular.Tabular + tabular.CSV dataset_id diff --git a/config/plugins/visualizations/scatterplot/config/scatterplot.xml b/config/plugins/visualizations/scatterplot/config/scatterplot.xml index 14004ceb1e5..d2649185929 100644 --- a/config/plugins/visualizations/scatterplot/config/scatterplot.xml +++ b/config/plugins/visualizations/scatterplot/config/scatterplot.xml @@ -5,6 +5,7 @@ HistoryDatasetAssociation tabular.Tabular + tabular.CSV dataset_id diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 4d540709735..4149b893a7c 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -695,6 +695,7 @@ class Registry( object ): 'scf' : binary.Scf(), 'sff' : binary.Sff(), 'tabular' : tabular.Tabular(), + 'csv' : tabular.CSV(), 'taxonomy' : tabular.Taxonomy(), 'txt' : data.Text(), 'wig' : interval.Wiggle(), @@ -727,6 +728,7 @@ class Registry( object ): 'scf' : 'application/octet-stream', 'sff' : 'application/octet-stream', 'tabular' : 'text/plain', + 'csv' : 'text/plain', 'taxonomy' : 'text/plain', 'txt' : 'text/plain', 'wig' : 'text/plain', @@ -761,7 +763,8 @@ class Registry( object ): tabular.Pileup(), interval.Interval(), tabular.Sam(), - tabular.Eland() + tabular.Eland(), + tabular.CSV() ] def get_converters_by_datatype( self, ext ): diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 39949091d34..53e194409c2 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -7,6 +7,7 @@ pkg_resources.require( "bx-python" ) import gzip import logging import os +import csv from cgi import escape from galaxy import util from galaxy.datatypes import data @@ -19,159 +20,90 @@ import dataproviders log = logging.getLogger(__name__) + @dataproviders.decorators.has_dataproviders -class Tabular( data.Text ): - """Tab delimited data""" +class TabularData( data.Text ): + """Generic tabular data""" edam_format = "format_3475" # All tabular data is chunkable. CHUNKABLE = True """Add metadata elements""" MetadataElement( name="comment_lines", default=0, desc="Number of comment lines", readonly=False, optional=True, no_value=0 ) + MetadataElement( name="data_lines", default=0, desc="Number of data lines", readonly=True, visible=False, optional=True, no_value=0 ) MetadataElement( name="columns", default=0, desc="Number of columns", readonly=True, visible=False, no_value=0 ) MetadataElement( name="column_types", default=[], desc="Column types", param=metadata.ColumnTypesParameter, readonly=True, visible=False, no_value=[] ) MetadataElement( name="column_names", default=[], desc="Column names", readonly=True, visible=False, optional=True, no_value=[] ) + MetadataElement( name="delimiter", default='\t', desc="Data delimiter", readonly=True, visible=False, optional=True, no_value=[] ) - def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 100000, max_guess_type_data_lines = None, **kwd ): - """ - Tries to determine the number of columns as well as those columns that - contain numerical values in the dataset. A skip parameter is used - because various tabular data types reuse this function, and their data - type classes are responsible to determine how many invalid comment - lines should be skipped. Using None for skip will cause skip to be - zero, but the first line will be processed as a header. A - max_data_lines parameter is used because various tabular data types - reuse this function, and their data type classes are responsible to - determine how many data lines should be processed to ensure that the - non-optional metadata parameters are properly set; if used, optional - metadata parameters will be set to None, unless the entire file has - already been read. Using None for max_data_lines will process all data - lines. + def set_peek( self, dataset, line_count=None, is_multi_byte=False): + super(TabularData, self).set_peek( dataset, line_count=line_count, is_multi_byte=is_multi_byte) + if dataset.metadata.comment_lines: + dataset.blurb = "%s, %s comments" % ( dataset.blurb, util.commaify( str( dataset.metadata.comment_lines ) ) ) - Items of interest: + def displayable( self, dataset ): + try: + return dataset.has_data() \ + and dataset.state == dataset.states.OK \ + and dataset.metadata.columns > 0 \ + and dataset.metadata.data_lines != 0 + except: + return False - 1. We treat 'overwrite' as always True (we always want to set tabular metadata when called). - 2. If a tabular file has no data, it will have one column of type 'str'. - 3. We used to check only the first 100 lines when setting metadata and this class's - set_peek() method read the entire file to determine the number of lines in the file. - Since metadata can now be processed on cluster nodes, we've merged the line count portion - of the set_peek() processing here, and we now check the entire contents of the file. - """ - # Store original skip value to check with later - requested_skip = skip - if skip is None: - skip = 0 - column_type_set_order = [ 'int', 'float', 'list', 'str' ] #Order to set column types in - default_column_type = column_type_set_order[-1] # Default column type is lowest in list - column_type_compare_order = list( column_type_set_order ) #Order to compare column types - column_type_compare_order.reverse() - def type_overrules_type( column_type1, column_type2 ): - if column_type1 is None or column_type1 == column_type2: - return False - if column_type2 is None: - return True - for column_type in column_type_compare_order: - if column_type1 == column_type: - return True - if column_type2 == column_type: - return False - #neither column type was found in our ordered list, this cannot happen - raise "Tried to compare unknown column types" - def is_int( column_text ): - try: - int( column_text ) - return True - except: - return False - def is_float( column_text ): - try: - float( column_text ) - return True - except: - if column_text.strip().lower() == 'na': - return True #na is special cased to be a float - return False - def is_list( column_text ): - return "," in column_text - def is_str( column_text ): - #anything, except an empty string, is True - if column_text == "": - return False - return True - is_column_type = {} #Dict to store column type string to checking function - for column_type in column_type_set_order: - is_column_type[column_type] = locals()[ "is_%s" % ( column_type ) ] - def guess_column_type( column_text ): - for column_type in column_type_set_order: - if is_column_type[column_type]( column_text ): - return column_type - return None - data_lines = 0 - comment_lines = 0 - column_types = [] - first_line_column_types = [default_column_type] # default value is one column of type str - if dataset.has_data(): - #NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default - dataset_fh = open( dataset.file_name ) - i = 0 - while True: - line = dataset_fh.readline() - if not line: break - line = line.rstrip( '\r\n' ) - if i < skip or not line or line.startswith( '#' ): - # We'll call blank lines comments - comment_lines += 1 - else: - data_lines += 1 - if max_guess_type_data_lines is None or data_lines <= max_guess_type_data_lines: - fields = line.split( '\t' ) - for field_count, field in enumerate( fields ): - if field_count >= len( column_types ): #found a previously unknown column, we append None - column_types.append( None ) - column_type = guess_column_type( field ) - if type_overrules_type( column_type, column_types[field_count] ): - column_types[field_count] = column_type - if i == 0 and requested_skip is None: - # This is our first line, people seem to like to upload files that have a header line, but do not - # start with '#' (i.e. all column types would then most likely be detected as str). We will assume - # that the first line is always a header (this was previous behavior - it was always skipped). When - # the requested skip is None, we only use the data from the first line if we have no other data for - # a column. This is far from perfect, as - # 1,2,3 1.1 2.2 qwerty - # 0 0 1,2,3 - # will be detected as - # "column_types": ["int", "int", "float", "list"] - # instead of - # "column_types": ["list", "float", "float", "str"] *** would seem to be the 'Truth' by manual - # observation that the first line should be included as data. The old method would have detected as - # "column_types": ["int", "int", "str", "list"] - first_line_column_types = column_types - column_types = [ None for col in first_line_column_types ] - if max_data_lines is not None and data_lines >= max_data_lines: - if dataset_fh.tell() != dataset.get_size(): - data_lines = None #Clear optional data_lines metadata value - comment_lines = None #Clear optional comment_lines metadata value; additional comment lines could appear below this point - break - i += 1 - dataset_fh.close() + def get_chunk(self, trans, dataset, chunk): + ck_index = int(chunk) + f = open(dataset.file_name) + f.seek(ck_index * trans.app.config.display_chunk_size) + # If we aren't at the start of the file, seek to next newline. Do this better eventually. + if f.tell() != 0: + cursor = f.read(1) + while cursor and cursor != '\n': + cursor = f.read(1) + ck_data = f.read(trans.app.config.display_chunk_size) + cursor = f.read(1) + while cursor and ck_data[-1] != '\n': + ck_data += cursor + cursor = f.read(1) + return dumps( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } ) + + def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd): + preview = util.string_as_bool( preview ) + if chunk: + return self.get_chunk(trans, dataset, chunk) + elif to_ext or not preview: + to_ext = to_ext or dataset.extension + return self._serve_raw(trans, dataset, to_ext) + elif dataset.metadata.columns > 50: + # Fancy tabular display is only suitable for datasets without an incredibly large number of columns. + # We should add a new datatype 'matrix', with its own draw method, suitable for this kind of data. + # For now, default to the old behavior, ugly as it is. Remove this after adding 'matrix'. + max_peek_size = 1000000 # 1 MB + if os.stat( dataset.file_name ).st_size < max_peek_size: + return open( dataset.file_name ) + else: + trans.response.set_content_type( "text/html" ) + return trans.stream_template_mako( "/dataset/large_file.mako", + truncated_data=open( dataset.file_name ).read(max_peek_size), + data=dataset) + else: + column_names = 'null' + if dataset.metadata.column_names: + column_names = dataset.metadata.column_names + elif hasattr(dataset.datatype, 'column_names'): + column_names = dataset.datatype.column_names + column_types = dataset.metadata.column_types + if not column_types: + column_types = [] + column_number = dataset.metadata.columns + if column_number is None: + column_number = 'null' + return trans.fill_template( "/dataset/tabular_chunked.mako", + dataset=dataset, + chunk=self.get_chunk(trans, dataset, 0), + column_number=column_number, + column_names=column_names, + column_types=column_types ) - #we error on the larger number of columns - #first we pad our column_types by using data from first line - if len( first_line_column_types ) > len( column_types ): - for column_type in first_line_column_types[len( column_types ):]: - column_types.append( column_type ) - #Now we fill any unknown (None) column_types with data from first line - for i in range( len( column_types ) ): - if column_types[i] is None: - if len( first_line_column_types ) <= i or first_line_column_types[i] is None: - column_types[i] = default_column_type - else: - column_types[i] = first_line_column_types[i] - # Set the discovered metadata values for the dataset - dataset.metadata.data_lines = data_lines - dataset.metadata.comment_lines = comment_lines - dataset.metadata.column_types = column_types - dataset.metadata.columns = len( column_types ) def make_html_table( self, dataset, **kwargs ): """Create HTML table, used for displaying peek""" out = [''] @@ -227,7 +159,7 @@ class Tabular( data.Text ): out.append( '' ) except Exception, exc: log.exception( 'make_html_peek_header failed on HDA %s' % dataset.id ) - raise Exception, "Can't create peek header %s" % str( exc ) + raise Exception( "Can't create peek header %s" % str( exc ) ) return "".join( out ) def make_html_peek_rows( self, dataset, skipchars=None, **kwargs ): @@ -244,7 +176,7 @@ class Tabular( data.Text ): if line.startswith( tuple( skipchars ) ): out.append( '' % escape( line ) ) elif line: - elems = line.split( '\t' ) + elems = line.split( dataset.metadata.delimiter ) # we may have an invalid comment line or invalid data if len( elems ) != columns: out.append( '' % escape( line ) ) @@ -255,106 +187,200 @@ class Tabular( data.Text ): out.append( '' ) except Exception, exc: log.exception( 'make_html_peek_rows failed on HDA %s' % dataset.id ) - raise Exception, "Can't create peek rows %s" % str( exc ) + raise Exception( "Can't create peek rows %s" % str( exc ) ) return "".join( out ) - def get_chunk(self, trans, dataset, chunk): - ck_index = int(chunk) - f = open(dataset.file_name) - f.seek(ck_index * trans.app.config.display_chunk_size) - # If we aren't at the start of the file, seek to next newline. Do this better eventually. - if f.tell() != 0: - cursor = f.read(1) - while cursor and cursor != '\n': - cursor = f.read(1) - ck_data = f.read(trans.app.config.display_chunk_size) - cursor = f.read(1) - while cursor and ck_data[-1] != '\n': - ck_data += cursor - cursor = f.read(1) - return dumps( { 'ck_data': util.unicodify( ck_data ), 'ck_index': ck_index + 1 } ) - - def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd): - preview = util.string_as_bool( preview ) - if chunk: - return self.get_chunk(trans, dataset, chunk) - elif to_ext or not preview: - to_ext = to_ext or dataset.extension - return self._serve_raw(trans, dataset, to_ext) - elif dataset.metadata.columns > 50: - #Fancy tabular display is only suitable for datasets without an incredibly large number of columns. - #We should add a new datatype 'matrix', with its own draw method, suitable for this kind of data. - #For now, default to the old behavior, ugly as it is. Remove this after adding 'matrix'. - max_peek_size = 1000000 # 1 MB - if os.stat( dataset.file_name ).st_size < max_peek_size: - return open( dataset.file_name ) - else: - trans.response.set_content_type( "text/html" ) - return trans.stream_template_mako( "/dataset/large_file.mako", - truncated_data = open( dataset.file_name ).read(max_peek_size), - data = dataset) - else: - column_names = 'null' - if dataset.metadata.column_names: - column_names = dataset.metadata.column_names - elif hasattr(dataset.datatype, 'column_names'): - column_names = dataset.datatype.column_names - column_types = dataset.metadata.column_types - if not column_types: - column_types = [] - column_number = dataset.metadata.columns - if column_number is None: - column_number = 'null' - return trans.fill_template( "/dataset/tabular_chunked.mako", - dataset = dataset, - chunk = self.get_chunk(trans, dataset, 0), - column_number = column_number, - column_names = column_names, - column_types = column_types ) - - def set_peek( self, dataset, line_count=None, is_multi_byte=False): - super(Tabular, self).set_peek( dataset, line_count=line_count, is_multi_byte=is_multi_byte) - if dataset.metadata.comment_lines: - dataset.blurb = "%s, %s comments" % ( dataset.blurb, util.commaify( str( dataset.metadata.comment_lines ) ) ) def display_peek( self, dataset ): """Returns formatted html of peek""" return self.make_html_table( dataset ) - def displayable( self, dataset ): - try: - return dataset.has_data() \ - and dataset.state == dataset.states.OK \ - and dataset.metadata.columns > 0 \ - and dataset.metadata.data_lines != 0 - except: - return False - def as_gbrowse_display_file( self, dataset, **kwd ): - return open( dataset.file_name ) - def as_ucsc_display_file( self, dataset, **kwd ): - return open( dataset.file_name ) # ------------- Dataproviders @dataproviders.decorators.dataprovider_factory( 'column', dataproviders.column.ColumnarDataProvider.settings ) def column_dataprovider( self, dataset, **settings ): """Uses column settings that are passed in""" dataset_source = dataproviders.dataset.DatasetDataProvider( dataset ) - return dataproviders.column.ColumnarDataProvider( dataset_source, **settings ) + delimiter = dataset.metadata.delimiter + return dataproviders.column.ColumnarDataProvider( dataset_source, deliminator=delimiter, **settings ) @dataproviders.decorators.dataprovider_factory( 'dataset-column', dataproviders.column.ColumnarDataProvider.settings ) def dataset_column_dataprovider( self, dataset, **settings ): """Attempts to get column settings from dataset.metadata""" - return dataproviders.dataset.DatasetColumnarDataProvider( dataset, **settings ) + delimiter = dataset.metadata.delimiter + return dataproviders.dataset.DatasetColumnarDataProvider( dataset, deliminator=delimiter, **settings ) @dataproviders.decorators.dataprovider_factory( 'dict', dataproviders.column.DictDataProvider.settings ) def dict_dataprovider( self, dataset, **settings ): """Uses column settings that are passed in""" dataset_source = dataproviders.dataset.DatasetDataProvider( dataset ) - return dataproviders.column.DictDataProvider( dataset_source, **settings ) + delimiter = dataset.metadata.delimiter + return dataproviders.column.DictDataProvider( dataset_source, deliminator=delimiter, **settings ) @dataproviders.decorators.dataprovider_factory( 'dataset-dict', dataproviders.column.DictDataProvider.settings ) def dataset_dict_dataprovider( self, dataset, **settings ): """Attempts to get column settings from dataset.metadata""" - return dataproviders.dataset.DatasetDictDataProvider( dataset, **settings ) + delimiter = dataset.metadata.delimiter + return dataproviders.dataset.DatasetDictDataProvider( dataset, deliminator=delimiter, **settings ) + + +@dataproviders.decorators.has_dataproviders +class Tabular( TabularData ): + """Tab delimited data""" + + def set_meta( self, dataset, overwrite=True, skip=None, max_data_lines=100000, max_guess_type_data_lines=None, **kwd ): + """ + Tries to determine the number of columns as well as those columns that + contain numerical values in the dataset. A skip parameter is used + because various tabular data types reuse this function, and their data + type classes are responsible to determine how many invalid comment + lines should be skipped. Using None for skip will cause skip to be + zero, but the first line will be processed as a header. A + max_data_lines parameter is used because various tabular data types + reuse this function, and their data type classes are responsible to + determine how many data lines should be processed to ensure that the + non-optional metadata parameters are properly set; if used, optional + metadata parameters will be set to None, unless the entire file has + already been read. Using None for max_data_lines will process all data + lines. + + Items of interest: + + 1. We treat 'overwrite' as always True (we always want to set tabular metadata when called). + 2. If a tabular file has no data, it will have one column of type 'str'. + 3. We used to check only the first 100 lines when setting metadata and this class's + set_peek() method read the entire file to determine the number of lines in the file. + Since metadata can now be processed on cluster nodes, we've merged the line count portion + of the set_peek() processing here, and we now check the entire contents of the file. + """ + # Store original skip value to check with later + requested_skip = skip + if skip is None: + skip = 0 + column_type_set_order = [ 'int', 'float', 'list', 'str' ] # Order to set column types in + default_column_type = column_type_set_order[-1] # Default column type is lowest in list + column_type_compare_order = list( column_type_set_order ) # Order to compare column types + column_type_compare_order.reverse() + + def type_overrules_type( column_type1, column_type2 ): + if column_type1 is None or column_type1 == column_type2: + return False + if column_type2 is None: + return True + for column_type in column_type_compare_order: + if column_type1 == column_type: + return True + if column_type2 == column_type: + return False + # neither column type was found in our ordered list, this cannot happen + raise "Tried to compare unknown column types" + + def is_int( column_text ): + try: + int( column_text ) + return True + except: + return False + + def is_float( column_text ): + try: + float( column_text ) + return True + except: + if column_text.strip().lower() == 'na': + return True # na is special cased to be a float + return False + + def is_list( column_text ): + return "," in column_text + + def is_str( column_text ): + # anything, except an empty string, is True + if column_text == "": + return False + return True + is_column_type = {} # Dict to store column type string to checking function + for column_type in column_type_set_order: + is_column_type[column_type] = locals()[ "is_%s" % ( column_type ) ] + + def guess_column_type( column_text ): + for column_type in column_type_set_order: + if is_column_type[column_type]( column_text ): + return column_type + return None + data_lines = 0 + comment_lines = 0 + column_types = [] + first_line_column_types = [default_column_type] # default value is one column of type str + if dataset.has_data(): + # NOTE: if skip > num_check_lines, we won't detect any metadata, and will use default + dataset_fh = open( dataset.file_name ) + i = 0 + while True: + line = dataset_fh.readline() + if not line: + break + line = line.rstrip( '\r\n' ) + if i < skip or not line or line.startswith( '#' ): + # We'll call blank lines comments + comment_lines += 1 + else: + data_lines += 1 + if max_guess_type_data_lines is None or data_lines <= max_guess_type_data_lines: + fields = line.split( '\t' ) + for field_count, field in enumerate( fields ): + if field_count >= len( column_types ): # found a previously unknown column, we append None + column_types.append( None ) + column_type = guess_column_type( field ) + if type_overrules_type( column_type, column_types[field_count] ): + column_types[field_count] = column_type + if i == 0 and requested_skip is None: + # This is our first line, people seem to like to upload files that have a header line, but do not + # start with '#' (i.e. all column types would then most likely be detected as str). We will assume + # that the first line is always a header (this was previous behavior - it was always skipped). When + # the requested skip is None, we only use the data from the first line if we have no other data for + # a column. This is far from perfect, as + # 1,2,3 1.1 2.2 qwerty + # 0 0 1,2,3 + # will be detected as + # "column_types": ["int", "int", "float", "list"] + # instead of + # "column_types": ["list", "float", "float", "str"] *** would seem to be the 'Truth' by manual + # observation that the first line should be included as data. The old method would have detected as + # "column_types": ["int", "int", "str", "list"] + first_line_column_types = column_types + column_types = [ None for col in first_line_column_types ] + if max_data_lines is not None and data_lines >= max_data_lines: + if dataset_fh.tell() != dataset.get_size(): + data_lines = None # Clear optional data_lines metadata value + comment_lines = None # Clear optional comment_lines metadata value; additional comment lines could appear below this point + break + i += 1 + dataset_fh.close() + + # we error on the larger number of columns + # first we pad our column_types by using data from first line + if len( first_line_column_types ) > len( column_types ): + for column_type in first_line_column_types[len( column_types ):]: + column_types.append( column_type ) + # Now we fill any unknown (None) column_types with data from first line + for i in range( len( column_types ) ): + if column_types[i] is None: + if len( first_line_column_types ) <= i or first_line_column_types[i] is None: + column_types[i] = default_column_type + else: + column_types[i] = first_line_column_types[i] + # Set the discovered metadata values for the dataset + dataset.metadata.data_lines = data_lines + dataset.metadata.comment_lines = comment_lines + dataset.metadata.column_types = column_types + dataset.metadata.columns = len( column_types ) + dataset.metadata.delimiter = '\t' + + def as_gbrowse_display_file( self, dataset, **kwd ): + return open( dataset.file_name ) + + def as_ucsc_display_file( self, dataset, **kwd ): + return open( dataset.file_name ) class Taxonomy( Tabular ): @@ -366,6 +392,7 @@ class Taxonomy( Tabular ): 'Superorder', 'Order', 'Suborder', 'Superfamily', 'Family', 'Subfamily', 'Tribe', 'Subtribe', 'Genus', 'Subgenus', 'Species', 'Subspecies' ] + def display_peek( self, dataset ): """Returns formated html of peek""" return Tabular.make_html_table( self, dataset, column_names=self.column_names ) @@ -384,6 +411,7 @@ class Sam( Tabular ): self.column_names = ['QNAME', 'FLAG', 'RNAME', 'POS', 'MAPQ', 'CIGAR', 'MRNM', 'MPOS', 'ISIZE', 'SEQ', 'QUAL', 'OPT' ] + def display_peek( self, dataset ): """Returns formated html of peek""" return Tabular.make_html_table( self, dataset, column_names=self.column_names ) @@ -423,7 +451,7 @@ class Sam( Tabular ): line = fh.readline() line = line.strip() if not line: - break #EOF + break # EOF if line: if line[0] != '@': linePieces = line.split('\t') @@ -447,7 +475,7 @@ class Sam( Tabular ): pass return False - def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 5, **kwd ): + def set_meta( self, dataset, overwrite=True, skip=None, max_data_lines=5, **kwd ): if dataset.has_data(): dataset_fh = open( dataset.file_name ) comment_lines = 0 @@ -487,9 +515,9 @@ class Sam( Tabular ): raise Exception('Result %s from %s' % (result, cmd)) merge = staticmethod(merge) - # ------------- Dataproviders + # Dataproviders # sam does not use '#' to indicate comments/headers - we need to strip out those headers from the std. providers - #TODO:?? seems like there should be an easier way to do this - metadata.comment_char? + # TODO:?? seems like there should be an easier way to do this - metadata.comment_char? @dataproviders.decorators.dataprovider_factory( 'line', dataproviders.line.FilteredLineDataProvider.settings ) def line_dataprovider( self, dataset, **settings ): settings[ 'comment_char' ] = '@' @@ -546,10 +574,10 @@ class Sam( Tabular ): settings[ 'comment_char' ] = '@' return dataproviders.dataset.GenomicRegionDataProvider( dataset, 2, 3, 3, True, **settings ) - #@dataproviders.decorators.dataprovider_factory( 'samtools' ) - #def samtools_dataprovider( self, dataset, **settings ): - # dataset_source = dataproviders.dataset.DatasetDataProvider( dataset ) - # return dataproviders.dataset.SamtoolsDataProvider( dataset_source, **settings ) + # @dataproviders.decorators.dataprovider_factory( 'samtools' ) + # def samtools_dataprovider( self, dataset, **settings ): + # dataset_source = dataproviders.dataset.DatasetDataProvider( dataset ) + # return dataproviders.dataset.SamtoolsDataProvider( dataset_source, **settings ) @dataproviders.decorators.has_dataproviders @@ -571,7 +599,7 @@ class Pileup( Tabular ): def display_peek( self, dataset ): """Returns formated html of peek""" - return Tabular.make_html_table( self, dataset, column_parameter_alias={'chromCol':'Chrom', 'startCol':'Start', 'baseCol':'Base'} ) + return Tabular.make_html_table( self, dataset, column_parameter_alias={'chromCol': 'Chrom', 'startCol': 'Start', 'baseCol': 'Base'} ) def repair_methods( self, dataset ): """Return options for removing errors along with a description""" @@ -612,7 +640,7 @@ class Pileup( Tabular ): except: return False - # ------------- Dataproviders + # Dataproviders @dataproviders.decorators.dataprovider_factory( 'genomic-region', dataproviders.dataset.GenomicRegionDataProvider.settings ) def genomic_region_dataprovider( self, dataset, **settings ): @@ -636,7 +664,7 @@ class Vcf( Tabular ): column_names = [ 'Chrom', 'Pos', 'ID', 'Ref', 'Alt', 'Qual', 'Filter', 'Info', 'Format', 'data' ] MetadataElement( name="columns", default=10, desc="Number of columns", readonly=True, visible=False ) - MetadataElement( name="column_types", default=['str','int','str','str','str','int','str','list','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False ) + MetadataElement( name="column_types", default=['str', 'int', 'str', 'str', 'str', 'int', 'str', 'list', 'str', 'str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False ) MetadataElement( name="viz_filter_cols", desc="Score column for visualization", default=[5], param=metadata.ColumnParameter, optional=True, multiple=True, visible=False ) MetadataElement( name="sample_names", default=[], desc="Sample names", readonly=True, visible=False, optional=True, no_value=[] ) @@ -662,7 +690,7 @@ class Vcf( Tabular ): # Found header line, get sample names. dataset.metadata.sample_names = line.split()[ 9: ] - # ------------- Dataproviders + # Dataproviders @dataproviders.decorators.dataprovider_factory( 'genomic-region', dataproviders.dataset.GenomicRegionDataProvider.settings ) def genomic_region_dataprovider( self, dataset, **settings ): @@ -685,6 +713,7 @@ class Eland( Tabular ): MetadataElement( name="reads", default=[], param=metadata.ListParameter, desc="Set of reads", readonly=True, visible=False, no_value=[] ) MetadataElement( name="lanes", default=[], param=metadata.ListParameter, desc="Set of lanes", readonly=True, visible=False, no_value=[] ) MetadataElement( name="barcodes", default=[], param=metadata.ListParameter, desc="Set of barcodes", readonly=True, visible=False, no_value=[] ) + def __init__(self, **kwd): """Initialize taxonomy datatype""" Tabular.__init__( self, **kwd ) @@ -693,6 +722,7 @@ class Eland( Tabular ): 'POSITION', 'STRAND', 'DESC', 'SRAS', 'PRAS', 'PART_CHROM' 'PART_CONTIG', 'PART_OFFSET', 'PART_STRAND', 'FILT' ] + def make_html_table( self, dataset, skipchars=None ): """Create HTML table, used for displaying peek""" if skipchars is None: @@ -702,11 +732,11 @@ class Eland( Tabular ): # Generate column header out.append( '' ) for i, name in enumerate( self.column_names ): - out.append( '' % ( str( i+1 ), name ) ) + out.append( '' % ( str( i + 1 ), name ) ) # This data type requires at least 11 columns in the data if dataset.metadata.columns - len( self.column_names ) > 0: for i in range( len( self.column_names ), dataset.metadata.columns ): - out.append( '' % str( i+1 ) ) + out.append( '' % str( i + 1 ) ) out.append( '' ) out.append( self.make_html_peek_rows( dataset, skipchars=skipchars ) ) out.append( '
%s
%s
%s.%s%s.%s%s%s
' ) @@ -714,6 +744,7 @@ class Eland( Tabular ): except Exception, exc: out = "Can't create peek %s" % exc return out + def sniff( self, filename ): """ Determines whether the file is in ELAND export format @@ -730,15 +761,15 @@ class Eland( Tabular ): try: compress = is_gzip(filename) if compress: - fh = gzip.GzipFile(filename, 'r') + fh = gzip.GzipFile(filename, 'r') else: - fh = open( filename ) + fh = open( filename ) count = 0 while True: line = fh.readline() line = line.strip() if not line: - break #EOF + break # EOF if line: linePieces = line.split('\t') if len(linePieces) != 22: @@ -767,32 +798,32 @@ class Eland( Tabular ): fh.close() return False - def set_meta( self, dataset, overwrite = True, skip = None, max_data_lines = 5, **kwd ): + def set_meta( self, dataset, overwrite=True, skip=None, max_data_lines=5, **kwd ): if dataset.has_data(): compress = is_gzip(dataset.file_name) if compress: - dataset_fh = gzip.GzipFile(dataset.file_name, 'r') + dataset_fh = gzip.GzipFile(dataset.file_name, 'r') else: - dataset_fh = open( dataset.file_name ) + dataset_fh = open( dataset.file_name ) lanes = {} tiles = {} barcodes = {} reads = {} - # # Should always read the entire file (until we devise a more clever way to pass metadata on) - #if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: - # # If the dataset is larger than optional_metadata, just count comment lines. - # dataset.metadata.data_lines = None - #else: - # # Otherwise, read the whole thing and set num data lines. + # # Should always read the entire file (until we devise a more clever way to pass metadata on) + # if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: + # # If the dataset is larger than optional_metadata, just count comment lines. + # dataset.metadata.data_lines = None + # else: + # # Otherwise, read the whole thing and set num data lines. for i, line in enumerate(dataset_fh): if line: linePieces = line.split('\t') if len(linePieces) != 22: - raise Exception('%s:%d:Corrupt line!' % (dataset.file_name,i)) - lanes[linePieces[2]]=1 - tiles[linePieces[3]]=1 - barcodes[linePieces[6]]=1 - reads[linePieces[7]]=1 + raise Exception('%s:%d:Corrupt line!' % (dataset.file_name, i)) + lanes[linePieces[2]] = 1 + tiles[linePieces[3]] = 1 + barcodes[linePieces[6]] = 1 + reads[linePieces[7]] = 1 pass dataset.metadata.data_lines = i + 1 dataset_fh.close() @@ -816,7 +847,81 @@ class FeatureLocationIndex( Tabular ): """ An index that stores feature locations in tabular format. """ - file_ext='fli' + file_ext = 'fli' MetadataElement( name="columns", default=2, desc="Number of columns", readonly=True, visible=False ) MetadataElement( name="column_types", default=['str', 'str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False, no_value=[] ) + +@dataproviders.decorators.has_dataproviders +class CSV( TabularData ): + """ + Delimiter-separated table data. + This includes CSV, TSV and other dialects understood by the + Python 'csv' module https://docs.python.org/2/library/csv.html + """ + delimiter = ',' + file_ext = 'csv' # File extension + peek_size = 1024 # File chunk used for sniffing CSV dialect + + def is_int( self, column_text ): + try: + int( column_text ) + return True + except: + return False + + def is_float( self, column_text ): + try: + float( column_text ) + return True + except: + if column_text.strip().lower() == 'na': + return True # na is special cased to be a float + return False + + def guess_type( self, text ): + if self.is_int(text): + return 'int' + if self.is_float(text): + return 'float' + else: + return 'str' + + def sniff( self, filename ): + """ Return True if if recognizes dialect and header. """ + if not csv.Sniffer().has_header(open(filename, 'r').read(self.peek_size)): + return False + # Fetch at least three consecutive lines to be reasonably sure + reader = csv.reader(open(filename, 'r')) + for i in range(0, 3): + reader.next() + return True + + def set_meta( self, dataset, **kwd ): + with open(dataset.file_name, 'r') as csvfile: + # Parse file + reader = csv.reader(csvfile) + data_row = None + header_row = None + try: + header_row = reader.next() + data_row = reader.next() + for row in reader: + pass + except csv.Error as e: + raise Exception('CSV reader error - line %d: %s' % (reader.line_num, e)) + + # Guess column types + if len(header_row) != len(data_row): + raise ('mismatching number of columns in header and data') + column_types = [] + for cell in data_row: + column_types.append(self.guess_type(cell)) + + # Set metadata + dataset.metadata.data_lines = reader.line_num - 1 + dataset.metadata.comment_lines = 1 + dataset.metadata.column_types = column_types + dataset.metadata.columns = len(header_row) + dataset.metadata.column_names = header_row + dataset.metadata.delimiter = reader.dialect.delimiter diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 69f3dce4f3b..2fc438181fd 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -48,12 +48,12 @@ class Json( Text ): else: with open(filename, "r") as fh: while True: - line = fh.readline() - line = line.strip() - if line: + # Grab first chunk of file and see if it looks like json. + start = fh.read(100).strip() + if start: # simple types are valid JSON as well - but would such a file # be interesting as JSON in Galaxy? - return line.startswith("[") or line.startswith("{") + return start.startswith("[") or start.startswith("{") return False def display_peek( self, dataset ): diff --git a/lib/galaxy/visualization/plugins/resource_parser.py b/lib/galaxy/visualization/plugins/resource_parser.py index e2f81ae2e15..314c71f7b48 100644 --- a/lib/galaxy/visualization/plugins/resource_parser.py +++ b/lib/galaxy/visualization/plugins/resource_parser.py @@ -23,7 +23,7 @@ class ResourceParser( object ): in a new dictionary. The keys used to store the new values can optionally be re-mapped to - new keys (e.g. dataset_id="NNN" -> hda=). + new keys (e.g. dataset_id="NNN" -> hda=). """ primitive_parsers = { 'str' : lambda param: galaxy.util.sanitize_html.sanitize_html( param, 'utf-8' ), diff --git a/lib/galaxy/web/base/controllers/admin.py b/lib/galaxy/web/base/controllers/admin.py index ecbda6a291c..6fc8648cb6c 100644 --- a/lib/galaxy/web/base/controllers/admin.py +++ b/lib/galaxy/web/base/controllers/admin.py @@ -32,10 +32,10 @@ class Admin( object ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) if trans.webapp.name == 'galaxy': - installed_repositories = trans.install_model.context.query( trans.install_model.ToolShedRepository ).first() + is_repo_installed = trans.install_model.context.query( trans.install_model.ToolShedRepository ).first() is not None installing_repository_ids = suc.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=True ) return trans.fill_template( '/webapps/galaxy/admin/index.mako', - installed_repositories=installed_repositories, + is_repo_installed=is_repo_installed, installing_repository_ids=installing_repository_ids, message=message, status=status ) @@ -50,7 +50,11 @@ class Admin( object ): message = escape( kwd.get( 'message', '' ) ) status = kwd.get( 'status', 'done' ) if trans.webapp.name == 'galaxy': + is_repo_installed = trans.install_model.context.query( trans.install_model.ToolShedRepository ).first() is not None + installing_repository_ids = suc.get_ids_of_tool_shed_repositories_being_installed( trans.app, as_string=True ) return trans.fill_template( '/webapps/galaxy/admin/center.mako', + is_repo_installed=is_repo_installed, + installing_repository_ids=installing_repository_ids, message=message, status=status ) else: diff --git a/lib/galaxy/web/base/interactive_environments.py b/lib/galaxy/web/base/interactive_environments.py index 5e12e166565..ae66fe148a6 100644 --- a/lib/galaxy/web/base/interactive_environments.py +++ b/lib/galaxy/web/base/interactive_environments.py @@ -8,6 +8,7 @@ import subprocess from galaxy.util.bunch import Bunch from galaxy import web from galaxy.managers import api_keys +from galaxy.tools.deps.docker_util import DockerVolume import logging log = logging.getLogger(__name__) @@ -154,7 +155,10 @@ class InteractiveEnviornmentRequest(object): .replace('${PORT}', str(self.attr.PORT)) return url - def docker_cmd(self, env_override={}): + def volume(self, host_path, container_path, **kwds): + return DockerVolume(host_path, container_path, **kwds) + + def docker_cmd(self, env_override={}, volumes=[]): """ Generate and return the docker command to execute """ @@ -162,16 +166,19 @@ class InteractiveEnviornmentRequest(object): conf = self.get_conf_dict() conf.update(env_override) env_str = ' '.join(['-e "%s=%s"' % (key.upper(), item) for key, item in conf.items()]) - return '%s run %s -d %s -p %s:%s -v "%s:/import/" %s' % \ + volume_str = ' '.join(['-v "%s"' % volume for volume in volumes]) + return '%s run %s -d %s -p %s:%s -v "%s:/import/" %s %s' % \ (self.attr.viz_config.get("docker", "command"), env_str, self.attr.viz_config.get("docker", "command_inject"), self.attr.PORT, self.attr.docker_port, - temp_dir, self.attr.viz_config.get("docker", "image")) + temp_dir, + volume_str, + self.attr.viz_config.get("docker", "image")) - def launch(self, raw_cmd=None, env_override={}): + def launch(self, raw_cmd=None, env_override={}, volumes=[]): if raw_cmd is None: - raw_cmd = self.docker_cmd(env_override=env_override) + raw_cmd = self.docker_cmd(env_override=env_override, volumes=volumes) log.info("Starting docker container for IE {0} with command [{1}]".format( self.attr.viz_id, raw_cmd diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py index 7523d00e567..01e146fc4af 100644 --- a/lib/tool_shed/metadata/metadata_generator.py +++ b/lib/tool_shed/metadata/metadata_generator.py @@ -630,10 +630,10 @@ class MetadataGenerator( object ): tool_requirements = [] for tool_requirement in tool.requirements: name = str( tool_requirement.name ) - type = str( tool_requirement.type ) + tool_type = str( tool_requirement.type ) version = str( tool_requirement.version ) if tool_requirement.version else None requirement_dict = dict( name=name, - type=type, + type=tool_type, version=version ) tool_requirements.append( requirement_dict ) # Handle tool.tests. @@ -665,9 +665,13 @@ class MetadataGenerator( object ): collection_type = values.collection_type metadata_display_value = "%s collection" % collection_type inputs.append( ( param_name, metadata_display_value ) ) - elif len( values ) == 1: - inputs.append( ( param_name, values[ 0 ] ) ) else: + try: + if len( values ) == 1: + inputs.append( ( param_name, values[ 0 ] ) ) + continue + except TypeError: + log.exception( 'Expected a list of values for tool "%s" parameter "%s", got %s: %s', tool.id, param_name, type( values ), values ) inputs.append( ( param_name, values ) ) outputs = [] for output in ttb.outputs: diff --git a/static/welcome.rst b/static/welcome.rst index 1fa6752002a..8125b4c6b9f 100644 --- a/static/welcome.rst +++ b/static/welcome.rst @@ -25,11 +25,11 @@ Need Help? * `Galaxy documentation`__ * `Galaxy wiki`__ -* Report bugs and problems to Galaxy mailing list: `galaxy-dev@bx.psu.edu`__ +* Report bugs and problems to Galaxy mailing list: `galaxy-dev@lists.galaxyproject.org`__ .. __: /static/help.html .. __: http://wiki.galaxyproject.org/ -.. __: mailto:galaxy-dev@bx.psu.edu +.. __: mailto:galaxy-dev@lists.galaxyproject.org Version: 2.1 Revision: $Rev$ diff --git a/templates/export_base.mako b/templates/export_base.mako index fe2d6326970..69da012634b 100644 --- a/templates/export_base.mako +++ b/templates/export_base.mako @@ -6,10 +6,12 @@ def inherit(context): if context.get('use_panels', False) == True: if context.get('webapp'): - webapp = context.get('webapp') + app_name = context.get('webapp') + elif context.get('app'): + app_name = context.get('app').name else: - webapp = 'galaxy' - return '/webapps/%s/base_panels.mako' % webapp + app_name = 'galaxy' + return '/webapps/%s/base_panels.mako' % app_name else: return '/base.mako' %> diff --git a/templates/form.mako b/templates/form.mako index 8d7c160e147..be0eab37d8a 100644 --- a/templates/form.mako +++ b/templates/form.mako @@ -2,10 +2,12 @@ def inherit(context): if context.get('use_panels'): if context.get('webapp'): - webapp = context.get('webapp') + app_name = context.get('webapp') + elif context.get('app'): + app_name = context.get('app').name else: - webapp = 'galaxy' - return '/webapps/%s/base_panels.mako' % webapp + app_name = 'galaxy' + return '/webapps/%s/base_panels.mako' % app_name else: return '/base.mako' %> diff --git a/templates/grid_base.mako b/templates/grid_base.mako index 98796d8c135..02cac200788 100644 --- a/templates/grid_base.mako +++ b/templates/grid_base.mako @@ -9,10 +9,12 @@ if context.get('use_panels'): if context.get('webapp'): - webapp = context.get('webapp') + app_name = context.get('webapp') + elif context.get('app'): + app_name = context.get('app').name else: - webapp = 'galaxy' - return '/webapps/%s/base_panels.mako' % webapp + app_name = 'galaxy' + return '/webapps/%s/base_panels.mako' % app_name else: return '/base.mako' %> diff --git a/templates/ind_share_base.mako b/templates/ind_share_base.mako index fd5d0edb5eb..341806e7f7e 100644 --- a/templates/ind_share_base.mako +++ b/templates/ind_share_base.mako @@ -6,10 +6,12 @@ def inherit(context): if context.get('use_panels'): if context.get('webapp'): - webapp = context.get('webapp') + app_name = context.get('webapp') + elif context.get('app'): + app_name = context.get('app').name else: - webapp = 'galaxy' - return '/webapps/%s/base_panels.mako' % webapp + app_name = 'galaxy' + return '/webapps/%s/base_panels.mako' % app_name else: return '/base.mako' %> diff --git a/templates/message.mako b/templates/message.mako index 59858ef16b8..a47b02c4741 100644 --- a/templates/message.mako +++ b/templates/message.mako @@ -2,10 +2,12 @@ def inherit(context): if context.get('use_panels'): if context.get('webapp'): - webapp = context.get('webapp') + app_name = context.get('webapp') + elif context.get('app'): + app_name = context.get('app').name else: - webapp = 'galaxy' - return '/webapps/%s/base_panels.mako' % webapp + app_name = 'galaxy' + return '/webapps/%s/base_panels.mako' % app_name else: return '/base.mako' %> diff --git a/templates/sharing_base.mako b/templates/sharing_base.mako index 4c452b9b9ea..21c459ab9b6 100644 --- a/templates/sharing_base.mako +++ b/templates/sharing_base.mako @@ -6,10 +6,12 @@ def inherit(context): if context.get('use_panels', False) == True: if context.get('webapp'): - webapp = context.get('webapp') + app_name = context.get('webapp') + elif context.get('app'): + app_name = context.get('app').name else: - webapp = 'galaxy' - return '/webapps/%s/base_panels.mako' % webapp + app_name = 'galaxy' + return '/webapps/%s/base_panels.mako' % app_name else: return '/base.mako' %> diff --git a/templates/webapps/galaxy/admin/center.mako b/templates/webapps/galaxy/admin/center.mako index f274a102515..a68bc4c102b 100644 --- a/templates/webapps/galaxy/admin/center.mako +++ b/templates/webapps/galaxy/admin/center.mako @@ -4,189 +4,107 @@ <%def name="title()">Galaxy Administration

Administration

- +Please visit the Galaxy administration hub to learn how to keep your Galaxy in best shape. %if message: ${render_msg( message, status )} %else: -

The menu on the left provides the following features

-
    -
  • Security - see the Data Security and Data Libraries section below for details -

    +

    Server

    • - Manage users - provides a view of the registered users and all groups and non-private roles associated - with each user. + Data types registry - See all datatypes available in this Galaxy.
    • -

    • - Manage groups - provides a view of all groups along with the members of the group and the roles associated with - each group (both private and non-private roles). The group names include a link to a page that allows you to manage the users and - roles that are associated with the group. + Data tables registry - See all data tables available in this Galaxy.
    • -

    • - Manage roles - provides a view of all non-private roles along with the role type, and the users and groups that - are associated with the role. The role names include a link to a page that allows you to manage the users and groups that are associated - with the role. The page also includes a view of the data library datasets that are associated with the role and the permissions applied - to each dataset. + Display applications - See all display applications configured in this Galaxy. +
    • +
    • + Manage jobs - Display all jobs that are currently not finished (i.e., their state is new, waiting, queued, or running). Administrators are able to cleanly stop long-running jobs.
    -
  • -

    -

  • Data -

    + +

    Tools and Tool Shed

    +
      + %if trans.app.tool_shed_registry and trans.app.tool_shed_registry.tool_sheds: +
    • + Search Tool Shed - Search and install new tools and other Galaxy utilities from the Tool Shed. See the tutorial. +
    • + %endif + %if tool_shed_repository_ids: +
    • + Monitor installing repositories - View the status of tools that are being currently installed. +
    • + %endif + %if is_repo_installed: +
    • + Manage installed tools - View and administer installed tools and utilities on this Galaxy. +
    • +
    • + Reset metadata - Select on which repositories you want to reset metadata. +
    • + %endif +
    • + Download local tool - Download a tarball with a tool from this Galaxy. +
    • +
    • + Reload a tool's configuration - Allows a new version of a tool to be loaded while the server is running. +
    • +
    • + Tool lineage - A view of a version lineages for all installed tools. Useful for debugging. +
    • +
    • + Review tool migration stages - See the list of migration stages that moved sets of tools from the distribution to the Tool Shed. +
    • +
    + +

    Security

    • - Manage data libraries - Data libraries enable a Galaxy administrator to upload datasets into a data library. Currently, - only administrators can create data libraries. -

      - When a data library is first created, it is considered "public" since it will be displayed in the "Data Libraries" view for any user, even - those that are not logged in. The Galaxy administrator can restrict access to a data library by associating roles with the data library's - "access library" permission. This permission will conservatively override the [dataset] "access" permission for the data library's contained - datasets. -

      - For example, if a data library's "access library" permission is associated with Role1 and the data library contains "public" datasets, the - data library will still only be displayed to those users that have Role1. However, if the data library's "access library" permission is - associated with both Role1 and Role2 and the data library contains datasets whose [dataset] "access" permission is associated with only Role1, - then users that have Role2 will be able to access the library, but will not see those contained datasets whose [dataset] "access" permission - is associated with only Role1. -

      - In addition to the "access library" permission, permission to perform the following functions on the data library (and its contents) can - be granted to users (a library item is one of: a data library, a library folder, a library dataset). -

      -

        -
      • add library item - Users that have the role can add library items to this data library or folder
      • -
      • modify library item - Users that have the role can modify this library item
      • -
      • manage library permissions - Users that have the role can manage permissions applied to this library item
      • -
      -

      - The default behavior is for no permissions to be applied to a data library item, but applied permissions are inherited downward (with the exception - of the "access library" permission, which is only available on the data library itself). Because of this, it is important to set desired permissions - on a new data library when it is created. When this is done, new folders and datasets added to the data library will automatically inherit those - permissions. In the same way, permissions can be applied to a folder, which will be automatically inherited by all contained datasets and sub-folders. -

      - The "Data Libraries" menu item allows users to access the datasets in a data library as long as they are not restricted from accessing them. - Importing a library dataset into a history will not make a copy of the dataset, but will be a "pointer" to the dataset on disk. This - approach allows for multiple users to use a single (possibly very large) dataset file. + Users - A view of all users and all groups and non-private roles associated with each user. +

    • +
    • + Groups - A view of all groups along with the members of the group and the roles associated with each group. +
    • +
    • + Roles - A view of all non-private roles along with the role type, and the users and groups that are associated with the role. + Also includes a view of the data library datasets that are associated with the role and the permissions applied to each dataset. +
    • +
    • + API keys - A view of all generated API keys with an option to re-generate. +
    • + %if trans.app.config.allow_user_impersonation: +
    • + Impersonate a user - Allows to view Galaxy as another user in order to help troubleshoot issues. +
    • + %endif +
    + +

    Data

    +
      + %if trans.app.config.enable_quotas: +
    • + Quotas - Manage user space quotas. See wiki for details. +
    • + %endif +
    • + Data libraries - Data libraries enable authorized Galaxy users to share datasets with other groups or users. Only administrators can create data libraries. See wiki for details and this page for security description. +
    • +
    • + Local data - Manage the reference (and other) data that is stored within Tool Data Tables. See wiki for details.
    -
  • -

    -

  • Server -

    +

    Form definitions

    • - Reload a tool's configuration - allows a new version of a tool to be loaded while the server is running -
    • -

      -

    • - Profile memory usage - measures system memory used for certain Galaxy functions -
    • -

      -

    • - Manage jobs - displays all jobs that are currently not finished (i.e., their state is new, waiting, queued, or - running). Administrators are able to cleanly stop long-running jobs. + Form definitions - Manage local form definitions.
    -
  • -

    -

  • Forms -

    To be completed -

  • -

    -

  • Sequencing Requests -

    To be completed -

  • -

    -

  • Cloud -

    To be completed -

  • -
-

-

Data Security and Data Libraries

-

- Security - Data security in Galaxy is a new feature, so familiarize yourself with the details which can be found - here or in our data security page. The data security - process incorporates users, groups and roles, and enables the application of certain permissions on datasets, specifically "access" - and "manage permissions". By default, the "manage permissions" permission is associated with the dataset owner's private role, and - the "access" permission is not set, making the dataset public. With these default permissions, users should not see any difference - in the way Galaxy has behaved in the past. -

    -
  • - Users - registered Galaxy users that have created a Galaxy account. Users can belong to groups and can - be associated with 1 or more roles. If a user is not authenticated during a Galaxy session, they will not have access to any - of the security features, and datasets they create during that session will have no permissions applied to them (i.e., they - will be considered "public", and no one will be allowed to change permissions on them). -
  • -

    -

  • - Groups - a set of 0 or more users which are considered members of the group. Groups can be associated with 0 - or more roles, simplifying the process of applying permissions to the data between a select group of users. -
  • -

    -

  • - Roles - associate users and groups with specific permissions on datasets. For example, users in groups A and B - can be associated with role C which gives them the "access" permission on datasets D, E and F. Roles have a type which is currently - one of the following: + +

    Sample tracking

    • - private - every user is associated automatically with their own private role. Administrators cannot - manage private roles. + Please see the sample tracking tutorial.
    • -
    • - user - this is currently not used, but eventually any registered user will be able to create a new role - and this will be its type. -
    • -
    • - sharing - a role created automatically during a Galaxy session that enables a user to share data with - another user. This can generally be considered a temporary role. -
    • -
    • admin - a role created by a Galaxy administrator.
    -
  • -

    -

  • - Dataset Permissions - applying the following permissions will to a dataset will result in the behavior described. -
      -
    • - access - users associated with the role can import this dataset into their history for analysis. -

      - If no roles with the "access" permission are associated with a dataset, the dataset is "public" and may be accessed by anyone - that can access the data library in which it is contained. See the Manage data libraries section above for - details. Public datasets contained in public data libraries will be accessible to all users (as well as anyone not logged in - during a Galaxy session) from the list of data libraries displayed when the "Data Libraries" menu item is selected. -

      -

      - Associating a dataset with a role that includes the "access" permission restricts the set of users that can access it. - For example, if 'Role A' includes the "access" permission and 'Role A' is associated with the dataset, only those users - and groups who are associated with 'Role A' may access the dataset. -

      -

      - If multiple roles that include the "access" permission are associated with a dataset, access to the dataset is derived - from the intersection of the users associated with the roles. For example, if 'Role A' and 'Role B' are associated with - a dataset, only those users and groups who are associated with both 'Role A' AND 'Role B' may access the dataset. When - the "access" permission is applied to a dataset, Galaxy checks to make sure that at least 1 user belongs to all groups and - roles associated with the "access" permission (otherwise the dataset would be restricted from everyone). -

      -

      - In order for a user to make a dataset private (i.e., only they can access it), they should associate the dataset with - their private role (the role identical to their Galaxy user name / email address). Associating additional roles that - include the "access" permission is not possible, since it would render the dataset inaccessible to everyone. -

      - To make a dataset private to themselves and one or more other users, the user can create a new role and associate the dataset - with that role, not their "private role". Galaxy makes this easy by telling the user they are about to share a private dataset - and giving them the option of doing so. If they respond positively, the sharing role is automatically created for them. -

      -

      - Private data (data associated with roles that include the "access" permission) must be made public in order to be used - with external applications like the "view at UCSC" link, or the "Perform genome analysis and prediction with EpiGRAPH" - tool. Being made publically accessible means removing the association of all roles that include the "access" permission - from the dataset. -

      -

    • -
    • manage permissions - Role members can manage the permissions applied to this dataset
    • -
    -
  • -
-
%endif diff --git a/templates/webapps/galaxy/admin/index.mako b/templates/webapps/galaxy/admin/index.mako index f5e6fd7c486..1d3fb604c5b 100644 --- a/templates/webapps/galaxy/admin/index.mako +++ b/templates/webapps/galaxy/admin/index.mako @@ -46,13 +46,43 @@
+
Server
+ +
+
Tools and Tool Shed
+
+
+ %if trans.app.tool_shed_registry and trans.app.tool_shed_registry.tool_sheds: + + %endif + %if installing_repository_ids: + + %endif + %if is_repo_installed: + + + %endif + + + + +
+
+
Security
- - - - + + + + %if trans.app.config.allow_user_impersonation: %endif @@ -63,55 +93,25 @@
%if trans.app.config.enable_quotas: - + %endif - - + +
-
Server
- - %if trans.app.tool_shed_registry and trans.app.tool_shed_registry.tool_sheds: -
-
Tool sheds
- - %endif -
Form Definitions
Sample Tracking
diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index d65ef7ca49c..196237fd414 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -213,6 +213,12 @@ Any data in tab delimited format (tabular) ----- +**Table (delimiter-separated)** + +Any delimiter-separated tabular data (CSV or TSV). + +----- + **Wig** The wiggle format is line-oriented. Wiggle data is preceded by a track definition line, which adds a number of options for controlling the default display of this track. diff --git a/tools/stats/gsummary.py b/tools/stats/gsummary.py index 334dda976d7..10b22a78f11 100755 --- a/tools/stats/gsummary.py +++ b/tools/stats/gsummary.py @@ -1,7 +1,12 @@ #!/usr/bin/env python import sys, re, tempfile -from rpy import * +try: + from rpy2.rpy_classic import * +except: + # RPy isn't maintained, and doesn't work with R>3.0, use it as a fallback + from rpy import * + # Older py compatibility try: set() @@ -87,11 +92,11 @@ def main(): stop_err( "Invalid column or column data values invalid for computation. See tool tips and syntax for data requirements." ) else: # summary function and return labels + set_default_mode( NO_CONVERSION ) summary_func = r( "function( x ) { c( sum=sum( as.numeric( x ), na.rm=T ), mean=mean( as.numeric( x ), na.rm=T ), stdev=sd( as.numeric( x ), na.rm=T ), quantile( as.numeric( x ), na.rm=TRUE ) ) }" ) headings = [ 'sum', 'mean', 'stdev', '0%', '25%', '50%', '75%', '100%' ] headings_str = "\t".join( headings ) - set_default_mode( NO_CONVERSION ) r_data_frame = r.read_table( tmp_file.name, header=True, sep="\t" ) outfile = open( outfile_name, 'w' ) @@ -105,7 +110,7 @@ def main(): stop_err( "Computation resulted in the following error: %s" % str( s ) ) summary = summary.as_py( BASIC_CONVERSION ) outfile.write( "#%s\n" % headings_str ) - outfile.write( "%s\n" % "\t".join( [ "%g" % ( summary[ k ] ) for k in headings ] ) ) + outfile.write( "%s\n" % "\t".join( [ "%g" % k for k in summary ] ) ) outfile.close() if skipped_lines: