diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 904750b5436..378cffe3e92 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -186,6 +186,6 @@ class UniverseApplication( object ): def configure_fluent_log( self ): if self.config.fluent_log: from galaxy.util.log.fluent_log import FluentTraceLogger - self.trace_logger = FluentTraceLogger( 'galaxy', self.config.fluent_host, self.config.fluent_port ) + self.trace_logger = FluentTraceLogger( 'galaxy', self.config.fluent_host, self.config.fluent_port ) else: self.trace_logger = None diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 0f3c7d92516..d2701555a8f 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -282,7 +282,7 @@ class Configuration( object ): self.biostar_url = kwargs.get( 'biostar_url', None ) self.biostar_key_name = kwargs.get( 'biostar_key_name', None ) self.biostar_key = kwargs.get( 'biostar_key', None ) - # Experimental: This will not be enabled by default and will hide + # Experimental: This will not be enabled by default and will hide # nonproduction code. # The api_folders refers to whether the API exposes the /folders section. self.api_folders = string_as_bool( kwargs.get( 'api_folders', False ) ) @@ -302,7 +302,7 @@ class Configuration( object ): @property def sentry_dsn_public( self ): """ - Sentry URL with private key removed for use in client side scripts, + Sentry URL with private key removed for use in client side scripts, sentry server will need to be configured to accept events """ if self.sentry_dsn: @@ -436,8 +436,8 @@ def configure_logging( config ): """ # Get root logger root = logging.getLogger() - # PasteScript will have already configured the logger if the - # 'loggers' section was found in the config file, otherwise we do + # PasteScript will have already configured the logger if the + # 'loggers' section was found in the config file, otherwise we do # some simple setup using the 'log_*' values from the config. if not config.global_conf_parser.has_section( "loggers" ): format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" ) diff --git a/lib/galaxy/datatypes/assembly.py b/lib/galaxy/datatypes/assembly.py index 3a0207a8ca5..3583e524a3d 100644 --- a/lib/galaxy/datatypes/assembly.py +++ b/lib/galaxy/datatypes/assembly.py @@ -168,7 +168,7 @@ class Velvet( Html ): def regenerate_primary_file(self,dataset): """ - cannot do this until we are setting metadata + cannot do this until we are setting metadata """ log.debug( "Velvet log info %s" % 'JJ regenerate_primary_file') gen_msg = '' diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 22fff082be8..fbca965b290 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -203,7 +203,7 @@ class Bam( Binary ): stderr = open( stderr_name ).read().strip() if stderr: if exit_code != 0: - shutil.rmtree( tmp_dir) #clean up + shutil.rmtree( tmp_dir) #clean up raise Exception, "Error Grooming BAM file contents: %s" % stderr else: print stderr @@ -231,7 +231,7 @@ class Bam( Binary ): stderr = open( stderr_name ).read().strip() if stderr: if exit_code != 0: - os.unlink( stderr_name ) #clean up + os.unlink( stderr_name ) #clean up raise Exception, "Error Setting BAM Metadata: %s" % stderr else: print stderr @@ -240,7 +240,7 @@ class Bam( Binary ): os.unlink( stderr_name ) def sniff( self, filename ): # BAM is compressed in the BGZF format, and must not be uncompressed in Galaxy. - # The first 4 bytes of any bam file is 'BAM\1', and the file is binary. + # The first 4 bytes of any bam file is 'BAM\1', and the file is binary. try: header = gzip.open( filename ).read(4) if binascii.b2a_hex( header ) == binascii.hexlify( 'BAM\1' ): @@ -250,7 +250,7 @@ class Bam( Binary ): return False def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: - dataset.peek = "Binary bam alignments file" + dataset.peek = "Binary bam alignments file" dataset.blurb = data.nice_size( dataset.get_size() ) else: dataset.peek = 'file does not exist' @@ -278,7 +278,7 @@ class Bam( Binary ): samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset ) settings[ 'comment_char' ] = '@' return dataproviders.line.RegexLineDataProvider( samtools_source, **settings ) - + @dataproviders.decorators.dataprovider_factory( 'column', dataproviders.column.ColumnarDataProvider.settings ) def column_dataprovider( self, dataset, **settings ): samtools_source = dataproviders.dataset.SamtoolsDataProvider( dataset ) @@ -352,7 +352,7 @@ class H5( Binary ): def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: - dataset.peek = "Binary h5 file" + dataset.peek = "Binary h5 file" dataset.blurb = data.nice_size( dataset.get_size() ) else: dataset.peek = 'file does not exist' @@ -372,7 +372,7 @@ class Scf( Binary ): def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: - dataset.peek = "Binary scf sequence file" + dataset.peek = "Binary scf sequence file" dataset.blurb = data.nice_size( dataset.get_size() ) else: dataset.peek = 'file does not exist' @@ -404,7 +404,7 @@ class Sff( Binary ): return False def set_peek( self, dataset, is_multi_byte=False ): if not dataset.dataset.purged: - dataset.peek = "Binary sff file" + dataset.peek = "Binary sff file" dataset.blurb = data.nice_size( dataset.get_size() ) else: dataset.peek = 'file does not exist' @@ -451,7 +451,7 @@ class BigWig(Binary): return dataset.peek except: return "Binary UCSC %s file (%s)" % ( self._name, data.nice_size( dataset.get_size() ) ) - + Binary.register_sniffable_binary_format("bigwig", "bigwig", BigWig) @@ -470,9 +470,9 @@ Binary.register_sniffable_binary_format("bigbed", "bigbed", BigBed) class TwoBit (Binary): """Class describing a TwoBit format nucleotide file""" - + file_ext = "twobit" - + def sniff(self, filename): try: # All twobit files start with a 16-byte header. If the file is smaller than 16 bytes, it's obviously not a valid twobit file. diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py index 76403f4b181..b20a1d889a5 100644 --- a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py @@ -23,13 +23,13 @@ class BedGraphReader: if not line: raise StopIteration() if line.isspace(): - continue + continue if line[0] == "#": continue if line[0].isalpha(): if line.startswith( "track" ) or line.startswith( "browser" ): continue - + feature = line.strip().split() chrom = feature[0] chrom_start = int(feature[1]) @@ -37,19 +37,19 @@ class BedGraphReader: score = float(feature[3]) return chrom, chrom_start, chrom_end, None, score def main(): - + input_fname = sys.argv[1] out_fname = sys.argv[2] - + reader = BedGraphReader( open( input_fname ) ) - + # Fill array from reader d = array_tree_dict_from_reader( reader, {}, block_size = BLOCK_SIZE ) - + for array_tree in d.itervalues(): array_tree.root.build_summary() - + FileArrayTreeDict.dict_to_file( d, open( out_fname, "w" ) ) -if __name__ == "__main__": +if __name__ == "__main__": main() \ No newline at end of file diff --git a/lib/galaxy/datatypes/converters/bgzip.py b/lib/galaxy/datatypes/converters/bgzip.py index 7a577129048..a419257a2b8 100644 --- a/lib/galaxy/datatypes/converters/bgzip.py +++ b/lib/galaxy/datatypes/converters/bgzip.py @@ -19,13 +19,13 @@ def main(): parser.add_option( '-P', '--preset', dest='preset' ) (options, args) = parser.parse_args() input_fname, output_fname = args - + tmpfile = tempfile.NamedTemporaryFile() sort_params = None - + if options.chrom_col and options.start_col and options.end_col: - sort_params = ["sort", - "-k%(i)s,%(i)s" % { 'i': options.chrom_col }, + sort_params = ["sort", + "-k%(i)s,%(i)s" % { 'i': options.chrom_col }, "-k%(i)i,%(i)in" % { 'i': options.start_col }, "-k%(i)i,%(i)in" % { 'i': options.end_col } ] @@ -40,9 +40,8 @@ def main(): after_sort = subprocess.Popen(sort_params, stdin=grepped.stdout, stderr=subprocess.PIPE, stdout=tmpfile ) grepped.stdout.close() output, err = after_sort.communicate() - + ctabix.tabix_compress(tmpfile.name, output_fname, force=True) - -if __name__ == "__main__": + +if __name__ == "__main__": main() - \ No newline at end of file diff --git a/lib/galaxy/datatypes/converters/fasta_to_len.py b/lib/galaxy/datatypes/converters/fasta_to_len.py index d604d8dd101..278a3449e5d 100644 --- a/lib/galaxy/datatypes/converters/fasta_to_len.py +++ b/lib/galaxy/datatypes/converters/fasta_to_len.py @@ -10,7 +10,7 @@ import sys, os assert sys.version_info[:2] >= ( 2, 4 ) def compute_fasta_length( fasta_file, out_file, keep_first_char, keep_first_word=False ): - + infile = fasta_file out = open( out_file, 'w') keep_first_char = int( keep_first_char ) diff --git a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py index 06654a17227..d9856ee9fc0 100644 --- a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py +++ b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py @@ -38,11 +38,11 @@ def main(): chunk_end = in_file.tell() out_file.write('{"start":"%s","end":"%s","sequences":"%s"},' % (chunk_begin, chunk_end, sequences)) chunk_begin = chunk_end - + chunk_end = in_file.tell() out_file.write('{"start":"%s","end":"%s","sequences":"%s"}' % (chunk_begin, chunk_end, (current_line % lines_per_chunk) / 4)) out_file.write(']}\n') - -if __name__ == "__main__": + +if __name__ == "__main__": main() diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py index 1b68b3f6a2a..b5d82ba81fb 100644 --- a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py @@ -7,7 +7,7 @@ the order should be: 1st line: @title_of_seq 2nd line: nucleotides 3rd line: +title_of_qualityscore (might be skipped) -4th line: quality scores +4th line: quality scores (in three forms: a. digits, b. ASCII codes, the first char as the coding base, c. ASCII codes without the first char.) Usage: @@ -52,4 +52,4 @@ def __main__(): outfile.close() -if __name__ == "__main__": __main__() \ No newline at end of file +if __name__ == "__main__": __main__() \ No newline at end of file diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py index 870a69fbfb4..c87228f842e 100644 --- a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py +++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py @@ -7,7 +7,7 @@ the order should be: 1st line: @title_of_seq 2nd line: nucleotides 3rd line: +title_of_qualityscore (might be skipped) -4th line: quality scores +4th line: quality scores (in three forms: a. digits, b. ASCII codes, the first char as the coding base, c. ASCII codes without the first char.) Usage: @@ -30,7 +30,7 @@ def __main__(): seq_title_startswith = '' default_coding_value = 64 fastq_block_lines = 0 - + for i, line in enumerate( file( infile_name ) ): line = line.rstrip() if not line or line.startswith( '#' ): @@ -52,7 +52,7 @@ def __main__(): if not qual_title_startswith: qual_title_startswith = line_startswith if line_startswith != qual_title_startswith: - stop_err( 'Invalid fastqsolexa format at line %d: %s.' % ( i + 1, line ) ) + stop_err( 'Invalid fastqsolexa format at line %d: %s.' % ( i + 1, line ) ) quality_title = line[1:] if quality_title and read_title != quality_title: stop_err( 'Invalid fastqsolexa format at line %d: sequence title "%s" differes from score title "%s".' % ( i + 1, read_title, quality_title ) ) @@ -67,15 +67,15 @@ def __main__(): # peek: ascii or digits? val = line.split()[0] - try: + try: check = int( val ) fastq_integer = True except: fastq_integer = False - + if fastq_integer: # digits qual = line - else: + else: # ascii quality_score_length = len( line ) if quality_score_length == read_length + 1: @@ -89,8 +89,7 @@ def __main__(): score = ord( char ) - quality_score_startswith # 64 qual = "%s%s " % ( qual, str( score ) ) outfile_score.write( '%s\n' % qual ) - + outfile_score.close() -if __name__ == "__main__": __main__() - \ No newline at end of file +if __name__ == "__main__": __main__() diff --git a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py index 2f682cc4dca..07b3631dc71 100644 --- a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py @@ -18,23 +18,22 @@ from bx.interval_index_file import Indexes def main(): # Arguments input_fname, out_fname = sys.argv[1:] - + # Do conversion. index = Indexes() offset = 0 reader_wrapper = GFFReaderWrapper( fileinput.FileInput( input_fname ), fix_strand=True ) - for feature in list( reader_wrapper ): + for feature in list( reader_wrapper ): # Add feature; index expects BED coordinates. if isinstance( feature, GenomicInterval ): convert_gff_coords_to_bed( feature ) index.add( feature.chrom, feature.start, feature.end, offset ) - + # Always increment offset, even if feature is not an interval and hence # not included in the index. offset += feature.raw_size index.write( open(out_fname, "w") ) - -if __name__ == "__main__": + +if __name__ == "__main__": main() - \ No newline at end of file diff --git a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py index f39c6be2b53..cfa48cedd28 100644 --- a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py @@ -1,62 +1,62 @@ -#!/usr/bin/env python -#Dan Blankenberg - -import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -import bx.intervals.io - -assert sys.version_info[:2] >= ( 2, 4 ) - -def stop_err( msg ): - sys.stderr.write( msg ) - sys.exit() - -def __main__(): - output_name = sys.argv[1] - input_name = sys.argv[2] - try: - chromCol = int( sys.argv[3] ) - 1 - except: - stop_err( "'%s' is an invalid chrom column, correct the column settings before attempting to convert the data format." % str( sys.argv[3] ) ) - try: - startCol = int( sys.argv[4] ) - 1 - except: - stop_err( "'%s' is an invalid start column, correct the column settings before attempting to convert the data format." % str( sys.argv[4] ) ) - try: - endCol = int( sys.argv[5] ) - 1 - except: - stop_err( "'%s' is an invalid end column, correct the column settings before attempting to convert the data format." % str( sys.argv[5] ) ) - try: - strandCol = int( sys.argv[6] ) - 1 - except: - strandCol = -1 - try: - nameCol = int( sys.argv[7] ) - 1 - except: - nameCol = -1 - skipped_lines = 0 - first_skipped_line = 0 - out = open( output_name,'w' ) - count = 0 - for count, region in enumerate( bx.intervals.io.NiceReaderWrapper( open( input_name, 'r' ), chrom_col=chromCol, start_col=startCol, end_col=endCol, strand_col=strandCol, fix_strand=True, return_header=False, return_comments=False ) ): - try: - if nameCol >= 0: - name = region.fields[nameCol] - else: - raise IndexError - except: - name = "region_%i" % count - try: - - out.write( "%s\t%i\t%i\t%s\t%i\t%s\n" % ( region.chrom, region.start, region.end, name, 0, region.strand ) ) - except: - skipped_lines += 1 - if not first_skipped_line: - first_skipped_line = count + 1 - out.close() - print "%i regions converted to BED." % ( count + 1 - skipped_lines ) - if skipped_lines > 0: - print "Skipped %d blank or invalid lines starting with line # %d." % ( skipped_lines, first_skipped_line ) - -if __name__ == "__main__": __main__() +#!/usr/bin/env python +#Dan Blankenberg + +import sys +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) +import bx.intervals.io + +assert sys.version_info[:2] >= ( 2, 4 ) + +def stop_err( msg ): + sys.stderr.write( msg ) + sys.exit() + +def __main__(): + output_name = sys.argv[1] + input_name = sys.argv[2] + try: + chromCol = int( sys.argv[3] ) - 1 + except: + stop_err( "'%s' is an invalid chrom column, correct the column settings before attempting to convert the data format." % str( sys.argv[3] ) ) + try: + startCol = int( sys.argv[4] ) - 1 + except: + stop_err( "'%s' is an invalid start column, correct the column settings before attempting to convert the data format." % str( sys.argv[4] ) ) + try: + endCol = int( sys.argv[5] ) - 1 + except: + stop_err( "'%s' is an invalid end column, correct the column settings before attempting to convert the data format." % str( sys.argv[5] ) ) + try: + strandCol = int( sys.argv[6] ) - 1 + except: + strandCol = -1 + try: + nameCol = int( sys.argv[7] ) - 1 + except: + nameCol = -1 + skipped_lines = 0 + first_skipped_line = 0 + out = open( output_name,'w' ) + count = 0 + for count, region in enumerate( bx.intervals.io.NiceReaderWrapper( open( input_name, 'r' ), chrom_col=chromCol, start_col=startCol, end_col=endCol, strand_col=strandCol, fix_strand=True, return_header=False, return_comments=False ) ): + try: + if nameCol >= 0: + name = region.fields[nameCol] + else: + raise IndexError + except: + name = "region_%i" % count + try: + + out.write( "%s\t%i\t%i\t%s\t%i\t%s\n" % ( region.chrom, region.start, region.end, name, 0, region.strand ) ) + except: + skipped_lines += 1 + if not first_skipped_line: + first_skipped_line = count + 1 + out.close() + print "%i regions converted to BED." % ( count + 1 - skipped_lines ) + if skipped_lines > 0: + print "Skipped %d blank or invalid lines starting with line # %d." % ( skipped_lines, first_skipped_line ) + +if __name__ == "__main__": __main__() diff --git a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py index 804378eb019..2fb612c4d66 100644 --- a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py @@ -64,7 +64,7 @@ def __main__(): force_num_columns = int( sys.argv[9] ) except: force_num_columns = None - + skipped_lines = 0 first_skipped_line = None out = open( output_name,'w' ) @@ -88,32 +88,32 @@ def __main__(): break #name (fields[3]) can be anything, no verification needed if len( fields ) > 4: - float( fields[4] ) #score - A score between 0 and 1000. If the track line useScore attribute is set to 1 for this annotation data set, the score value will determine the level of gray in which this feature is displayed (higher numbers = darker gray). + float( fields[4] ) #score - A score between 0 and 1000. If the track line useScore attribute is set to 1 for this annotation data set, the score value will determine the level of gray in which this feature is displayed (higher numbers = darker gray). if len( fields ) > 5: - assert fields[5] in [ '+', '-' ], 'Invalid strand' #strand - Defines the strand - either '+' or '-'. + assert fields[5] in [ '+', '-' ], 'Invalid strand' #strand - Defines the strand - either '+' or '-'. if len( fields ) > 6: - int( fields[6] ) #thickStart - The starting position at which the feature is drawn thickly (for example, the start codon in gene displays). + int( fields[6] ) #thickStart - The starting position at which the feature is drawn thickly (for example, the start codon in gene displays). if len( fields ) > 7: - int( fields[7] ) #thickEnd - The ending position at which the feature is drawn thickly (for example, the stop codon in gene displays). - if len( fields ) > 8: + int( fields[7] ) #thickEnd - The ending position at which the feature is drawn thickly (for example, the stop codon in gene displays). + if len( fields ) > 8: if fields[8] != '0': #itemRgb - An RGB value of the form R,G,B (e.g. 255,0,0). If the track line itemRgb attribute is set to "On", this RBG value will determine the display color of the data contained in this BED line. NOTE: It is recommended that a simple color scheme (eight colors or less) be used with this attribute to avoid overwhelming the color resources of the Genome Browser and your Internet browser. fields2 = fields[8].split( ',' ) assert len( fields2 ) == 3, 'RGB value must be 0 or have length of 3' for field in fields2: int( field ) #rgb values are integers if len( fields ) > 9: - int( fields[9] ) #blockCount - The number of blocks (exons) in the BED line. + int( fields[9] ) #blockCount - The number of blocks (exons) in the BED line. if len( fields ) > 10: - if fields[10] != ',': #blockSizes - A comma-separated list of the block sizes. The number of items in this list should correspond to blockCount. + if fields[10] != ',': #blockSizes - A comma-separated list of the block sizes. The number of items in this list should correspond to blockCount. fields2 = fields[10].rstrip( "," ).split( "," ) #remove trailing comma and split on comma - for field in fields2: + for field in fields2: int( field ) if len( fields ) > 11: - if fields[11] != ',': #blockStarts - A comma-separated list of block starts. All of the blockStart positions should be calculated relative to chromStart. The number of items in this list should correspond to blockCount. + if fields[11] != ',': #blockStarts - A comma-separated list of block starts. All of the blockStart positions should be calculated relative to chromStart. The number of items in this list should correspond to blockCount. fields2 = fields[11].rstrip( "," ).split( "," ) #remove trailing comma and split on comma for field in fields2: int( field ) - except: + except: strict_bed = False break if force_num_columns is not None and len( fields ) != force_num_columns: @@ -122,7 +122,7 @@ def __main__(): else: strict_bed = False out.close() - + if not strict_bed: skipped_lines = 0 first_skipped_line = None diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.py b/lib/galaxy/datatypes/converters/interval_to_coverage.py index a73d25a4ddf..07093be0adc 100644 --- a/lib/galaxy/datatypes/converters/interval_to_coverage.py +++ b/lib/galaxy/datatypes/converters/interval_to_coverage.py @@ -50,12 +50,12 @@ def main( interval, coverage ): forward = forward_covs[partition] reverse = reverse_covs[partition] if forward+reverse > 0: - coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), + coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), forward=forward, reverse=reverse) partitions = [] forward_covs = [] reverse_covs = [] - + start_index = bisect(partitions, record.start) forward = int(record.strand == "+") reverse = int(record.strand == "-") @@ -74,43 +74,43 @@ def main( interval, coverage ): partitions.insert(end_index, record.end) forward_covs.insert(end_index, forward_covs[end_index-1] - forward ) reverse_covs.insert(end_index, reverse_covs[end_index-1] - reverse ) - + if partitions: for partition in xrange(0, start_index): forward = forward_covs[partition] reverse = reverse_covs[partition] if forward+reverse > 0: - coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), + coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), forward=forward, reverse=reverse) partitions = partitions[start_index:] forward_covs = forward_covs[start_index:] reverse_covs = reverse_covs[start_index:] - + lastchrom = chrom - + # Finish the last chromosome if partitions: for partition in xrange(0, len(partitions)-1): forward = forward_covs[partition] reverse = reverse_covs[partition] if forward+reverse > 0: - coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), + coverage.write(chrom=chrom, position=xrange(partitions[partition],partitions[partition+1]), forward=forward, reverse=reverse) - + class CoverageWriter( object ): def __init__( self, out_stream=None, chromCol=0, positionCol=1, forwardCol=2, reverseCol=3 ): self.out_stream = out_stream self.reverseCol = reverseCol self.nlines = 0 - positions = {str(chromCol):'%(chrom)s', - str(positionCol):'%(position)d', - str(forwardCol):'%(forward)d', + positions = {str(chromCol):'%(chrom)s', + str(positionCol):'%(position)d', + str(forwardCol):'%(forward)d', str(reverseCol):'%(reverse)d'} - if reverseCol < 0: + if reverseCol < 0: self.template = "%(0)s\t%(1)s\t%(2)s\n" % positions else: self.template = "%(0)s\t%(1)s\t%(2)s\t%(3)s\n" % positions - + def write(self, **kwargs ): if self.reverseCol < 0: kwargs['forward'] += kwargs['reverse'] posgen = kwargs['position'] @@ -121,12 +121,12 @@ class CoverageWriter( object ): def close(self): self.out_stream.flush() self.out_stream.close() - + if __name__ == "__main__": options, args = doc_optparse.parse( __doc__ ) try: chr_col_1, start_col_1, end_col_1, strand_col_1 = [int(x)-1 for x in options.cols1.split(',')] - chr_col_2, position_col_2, forward_col_2, reverse_col_2 = [int(x)-1 for x in options.cols2.split(',')] + chr_col_2, position_col_2, forward_col_2, reverse_col_2 = [int(x)-1 for x in options.cols2.split(',')] in_fname, out_fname = args except: doc_optparse.exception() @@ -141,7 +141,7 @@ if __name__ == "__main__": chromCol = chr_col_2, positionCol = position_col_2, forwardCol = forward_col_2, reverseCol = reverse_col_2, ) temp_file.seek(0) - interval = io.NiceReaderWrapper( temp_file, + interval = io.NiceReaderWrapper( temp_file, chrom_col=chr_col_1, start_col=start_col_1, end_col=end_col_1, diff --git a/lib/galaxy/datatypes/converters/interval_to_fli.py b/lib/galaxy/datatypes/converters/interval_to_fli.py index 8448ee62d29..22b429d1945 100644 --- a/lib/galaxy/datatypes/converters/interval_to_fli.py +++ b/lib/galaxy/datatypes/converters/interval_to_fli.py @@ -78,13 +78,13 @@ def main(): if len( fields ) < 4: continue - # Process line + # Process line name_loc_dict[ fields[3] ] = { 'contig': fields[0], 'start': int( fields[1] ), 'end': int ( fields[2] ) } - + # Create sorted list of entries. out = open( out_fname, 'w' ) max_len = 0 @@ -95,7 +95,7 @@ def main(): if len( entry ) > max_len: max_len = len( entry ) entries.append( entry ) - + # Write padded entries. out.write( str( max_len + 1 ).ljust( max_len ) + '\n' ) for entry in entries: diff --git a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py index 39705f44004..13fb2e53c73 100644 --- a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py @@ -20,20 +20,19 @@ def main(): parser.add_option( '-P', '--preset', dest='preset' ) (options, args) = parser.parse_args() input_fname, index_fname, out_fname = args - + # Create index. if options.preset: # Preset type. - ctabix.tabix_index(filename=index_fname, preset=options.preset, keep_original=True, + ctabix.tabix_index(filename=index_fname, preset=options.preset, keep_original=True, already_compressed=True, index_filename=out_fname) else: # For interval files; column indices are 0-based. - ctabix.tabix_index(filename=index_fname, seq_col=(options.chrom_col - 1), - start_col=(options.start_col - 1), end_col=(options.end_col - 1), + ctabix.tabix_index(filename=index_fname, seq_col=(options.chrom_col - 1), + start_col=(options.start_col - 1), end_col=(options.end_col - 1), keep_original=True, already_compressed=True, index_filename=out_fname) if os.path.getsize(index_fname) == 0: sys.stderr.write("The converted tabix index file is empty, meaning the input data is invalid.") - -if __name__ == "__main__": + +if __name__ == "__main__": main() - \ No newline at end of file diff --git a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py index 8e595a0955c..3ec59727b68 100644 --- a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py @@ -1,110 +1,110 @@ -# for rgenetics - lped to fbat -# recode to numeric fbat version -# much slower so best to always -# use numeric alleles internally - -import sys,os,time - - -prog = os.path.split(sys.argv[0])[-1] -myversion = 'Oct 10 2009' - -galhtmlprefix = """ - - - - - - - - - -
-""" - -def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) - - -def rgConv(inpedfilepath,outhtmlname,outfilepath): - """convert linkage ped/map to fbat""" - recode={'A':'1','C':'2','G':'3','T':'4','N':'0','0':'0','1':'1','2':'2','3':'3','4':'4'} - basename = os.path.split(inpedfilepath)[-1] # get basename - inmap = '%s.map' % inpedfilepath - inped = '%s.ped' % inpedfilepath - outf = '%s.ped' % basename # note the fbat exe insists that this is the extension for the ped data - outfpath = os.path.join(outfilepath,outf) # where to write the fbat format file to - try: - mf = file(inmap,'r') - except: - sys.stderr.write('%s cannot open inmap file %s - do you have permission?\n' % (prog,inmap)) - sys.exit(1) - try: - rsl = [x.split()[1] for x in mf] - except: - sys.stderr.write('## cannot parse %s' % inmap) - sys.exit(1) - try: - os.makedirs(outfilepath) - except: - pass # already exists - head = ' '.join(rsl) # list of rs numbers - # TODO add anno to rs but fbat will prolly barf? - pedf = file(inped,'r') - o = file(outfpath,'w',2**20) - o.write(head) - o.write('\n') - for i,row in enumerate(pedf): - if i == 0: - lrow = row.split() - try: - x = [int(x) for x in lrow[10:50]] # look for non numeric codes - except: - dorecode = 1 - if dorecode: - lrow = row.strip().split() - p = lrow[:6] - g = lrow[6:] - gc = [recode.get(x,'0') for x in g] - lrow = p+gc - row = '%s\n' % ' '.join(lrow) - o.write(row) - o.close() - - -def main(): - """call fbater - need to work with rgenetics composite datatypes - so in and out are html files with data in extrafiles path - rg_convert_lped_fped.py '$input1/$input1.metadata.base_name' - '$output1' '$output1.extra_files_path' - - """ - nparm = 3 - if len(sys.argv) < nparm: - sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog,sys.argv,nparm)) - sys.exit(1) - inpedfilepath = sys.argv[1] - outhtmlname = sys.argv[2] - outfilepath = sys.argv[3] - try: - os.makedirs(outfilepath) - except: - pass - rgConv(inpedfilepath,outhtmlname,outfilepath) - f = file(outhtmlname,'w') - f.write(galhtmlprefix % prog) - flist = os.listdir(outfilepath) - print '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info - f.write('
## Rgenetics: http://rgenetics.org Galaxy Tools %s %s\n
    ' % (prog,timenow())) - for i, data in enumerate( flist ): - f.write('
  1. %s
  2. \n' % (os.path.split(data)[-1],os.path.split(data)[-1])) - f.write("
") - f.close() - - - -if __name__ == "__main__": - main() +# for rgenetics - lped to fbat +# recode to numeric fbat version +# much slower so best to always +# use numeric alleles internally + +import sys,os,time + + +prog = os.path.split(sys.argv[0])[-1] +myversion = 'Oct 10 2009' + +galhtmlprefix = """ + + + + + + + + + +
+""" + +def timenow(): + """return current time as a string + """ + return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + + +def rgConv(inpedfilepath,outhtmlname,outfilepath): + """convert linkage ped/map to fbat""" + recode={'A':'1','C':'2','G':'3','T':'4','N':'0','0':'0','1':'1','2':'2','3':'3','4':'4'} + basename = os.path.split(inpedfilepath)[-1] # get basename + inmap = '%s.map' % inpedfilepath + inped = '%s.ped' % inpedfilepath + outf = '%s.ped' % basename # note the fbat exe insists that this is the extension for the ped data + outfpath = os.path.join(outfilepath,outf) # where to write the fbat format file to + try: + mf = file(inmap,'r') + except: + sys.stderr.write('%s cannot open inmap file %s - do you have permission?\n' % (prog,inmap)) + sys.exit(1) + try: + rsl = [x.split()[1] for x in mf] + except: + sys.stderr.write('## cannot parse %s' % inmap) + sys.exit(1) + try: + os.makedirs(outfilepath) + except: + pass # already exists + head = ' '.join(rsl) # list of rs numbers + # TODO add anno to rs but fbat will prolly barf? + pedf = file(inped,'r') + o = file(outfpath,'w',2**20) + o.write(head) + o.write('\n') + for i,row in enumerate(pedf): + if i == 0: + lrow = row.split() + try: + x = [int(x) for x in lrow[10:50]] # look for non numeric codes + except: + dorecode = 1 + if dorecode: + lrow = row.strip().split() + p = lrow[:6] + g = lrow[6:] + gc = [recode.get(x,'0') for x in g] + lrow = p+gc + row = '%s\n' % ' '.join(lrow) + o.write(row) + o.close() + + +def main(): + """call fbater + need to work with rgenetics composite datatypes + so in and out are html files with data in extrafiles path + rg_convert_lped_fped.py '$input1/$input1.metadata.base_name' + '$output1' '$output1.extra_files_path' + + """ + nparm = 3 + if len(sys.argv) < nparm: + sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog,sys.argv,nparm)) + sys.exit(1) + inpedfilepath = sys.argv[1] + outhtmlname = sys.argv[2] + outfilepath = sys.argv[3] + try: + os.makedirs(outfilepath) + except: + pass + rgConv(inpedfilepath,outhtmlname,outfilepath) + f = file(outhtmlname,'w') + f.write(galhtmlprefix % prog) + flist = os.listdir(outfilepath) + print '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info + f.write('
## Rgenetics: http://rgenetics.org Galaxy Tools %s %s\n
    ' % (prog,timenow())) + for i, data in enumerate( flist ): + f.write('
  1. %s
  2. \n' % (os.path.split(data)[-1],os.path.split(data)[-1])) + f.write("
") + f.close() + + + +if __name__ == "__main__": + main() diff --git a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py index 2c439b9f739..889b726124e 100644 --- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py +++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py @@ -1,110 +1,110 @@ -# for rgenetics - lped to pbed -# where to stop with converters -# pbed might be central -# eg lped/eigen/fbat/snpmatrix all to pbed -# and pbed to lped/eigen/fbat/snpmatrix ? -# that's a lot of converters -import sys,os,time,subprocess - - -prog = os.path.split(sys.argv[0])[-1] -myversion = 'Oct 10 2009' - -galhtmlprefix = """ - - - - - - - - - -
-""" - -def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) - -def getMissval(inped=''): - """ - read some lines...ugly hack - try to guess missing value - should be N or 0 but might be . or - - """ - commonmissvals = {'N':'N','0':'0','n':'n','9':'9','-':'-','.':'.'} - try: - f = file(inped,'r') - except: - return None # signal no in file - missval = None - while missval == None: # doggedly continue until we solve the mystery - try: - l = f.readline() - except: - break - ll = l.split()[6:] # ignore pedigree stuff - for c in ll: - if commonmissvals.get(c,None): - missval = c - f.close() - return missval - if not missval: - missval = 'N' # punt - close(f) - return missval - -def rgConv(inpedfilepath,outhtmlname,outfilepath,plink): - """ - """ - pedf = '%s.ped' % inpedfilepath - basename = os.path.split(inpedfilepath)[-1] # get basename - outroot = os.path.join(outfilepath,basename) - missval = getMissval(inped = pedf) - if not missval: - print '### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf - missval = '0' - cl = '%s --noweb --file %s --make-bed --out %s --missing-genotype %s' % (plink,inpedfilepath,outroot,missval) - p = subprocess.Popen(cl,shell=True,cwd=outfilepath) - retval = p.wait() # run plink - - - - -def main(): - """ - need to work with rgenetics composite datatypes - so in and out are html files with data in extrafiles path - lped_to_pbed_converter.py '$input1/$input1.metadata.base_name' - '$output1' '$output1.extra_files_path' '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink' - - """ - nparm = 4 - if len(sys.argv) < nparm: - sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog,sys.argv,nparm)) - sys.exit(1) - inpedfilepath = sys.argv[1] - outhtmlname = sys.argv[2] - outfilepath = sys.argv[3] - try: - os.makedirs(outfilepath) - except: - pass - plink = sys.argv[4] - rgConv(inpedfilepath,outhtmlname,outfilepath,plink) - f = file(outhtmlname,'w') - f.write(galhtmlprefix % prog) - flist = os.listdir(outfilepath) - s = '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info - print s - f.write('
%s\n
    ' % (s)) - for i, data in enumerate( flist ): - f.write('
  1. %s
  2. \n' % (os.path.split(data)[-1],os.path.split(data)[-1])) - f.write("
") - f.close() - - - -if __name__ == "__main__": - main() +# for rgenetics - lped to pbed +# where to stop with converters +# pbed might be central +# eg lped/eigen/fbat/snpmatrix all to pbed +# and pbed to lped/eigen/fbat/snpmatrix ? +# that's a lot of converters +import sys,os,time,subprocess + + +prog = os.path.split(sys.argv[0])[-1] +myversion = 'Oct 10 2009' + +galhtmlprefix = """ + + + + + + + + + +
+""" + +def timenow(): + """return current time as a string + """ + return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + +def getMissval(inped=''): + """ + read some lines...ugly hack - try to guess missing value + should be N or 0 but might be . or - + """ + commonmissvals = {'N':'N','0':'0','n':'n','9':'9','-':'-','.':'.'} + try: + f = file(inped,'r') + except: + return None # signal no in file + missval = None + while missval == None: # doggedly continue until we solve the mystery + try: + l = f.readline() + except: + break + ll = l.split()[6:] # ignore pedigree stuff + for c in ll: + if commonmissvals.get(c,None): + missval = c + f.close() + return missval + if not missval: + missval = 'N' # punt + close(f) + return missval + +def rgConv(inpedfilepath,outhtmlname,outfilepath,plink): + """ + """ + pedf = '%s.ped' % inpedfilepath + basename = os.path.split(inpedfilepath)[-1] # get basename + outroot = os.path.join(outfilepath,basename) + missval = getMissval(inped = pedf) + if not missval: + print '### lped_to_pbed_converter.py cannot identify missing value in %s' % pedf + missval = '0' + cl = '%s --noweb --file %s --make-bed --out %s --missing-genotype %s' % (plink,inpedfilepath,outroot,missval) + p = subprocess.Popen(cl,shell=True,cwd=outfilepath) + retval = p.wait() # run plink + + + + +def main(): + """ + need to work with rgenetics composite datatypes + so in and out are html files with data in extrafiles path + lped_to_pbed_converter.py '$input1/$input1.metadata.base_name' + '$output1' '$output1.extra_files_path' '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink' + + """ + nparm = 4 + if len(sys.argv) < nparm: + sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog,sys.argv,nparm)) + sys.exit(1) + inpedfilepath = sys.argv[1] + outhtmlname = sys.argv[2] + outfilepath = sys.argv[3] + try: + os.makedirs(outfilepath) + except: + pass + plink = sys.argv[4] + rgConv(inpedfilepath,outhtmlname,outfilepath,plink) + f = file(outhtmlname,'w') + f.write(galhtmlprefix % prog) + flist = os.listdir(outfilepath) + s = '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info + print s + f.write('
%s\n
    ' % (s)) + for i, data in enumerate( flist ): + f.write('
  1. %s
  2. \n' % (os.path.split(data)[-1],os.path.split(data)[-1])) + f.write("
") + f.close() + + + +if __name__ == "__main__": + main() diff --git a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py index 8f96f618cb5..df62e347242 100644 --- a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py @@ -1,32 +1,32 @@ -#!/usr/bin/env python -#Dan Blankenberg - -import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -import bx.align.maf -from galaxy.tools.util import maf_utilities - -assert sys.version_info[:2] >= ( 2, 4 ) - -def __main__(): - output_name = sys.argv.pop(1) - input_name = sys.argv.pop(1) - out = open( output_name, 'w' ) - count = 0 - for count, block in enumerate( bx.align.maf.Reader( open( input_name, 'r' ) ) ): +#!/usr/bin/env python +#Dan Blankenberg + +import sys +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) +import bx.align.maf +from galaxy.tools.util import maf_utilities + +assert sys.version_info[:2] >= ( 2, 4 ) + +def __main__(): + output_name = sys.argv.pop(1) + input_name = sys.argv.pop(1) + out = open( output_name, 'w' ) + count = 0 + for count, block in enumerate( bx.align.maf.Reader( open( input_name, 'r' ) ) ): spec_counts = {} - for c in block.components: + for c in block.components: spec, chrom = maf_utilities.src_split( c.src ) if spec not in spec_counts: spec_counts[ spec ] = 0 else: - spec_counts[ spec ] += 1 - out.write( "%s\n" % maf_utilities.get_fasta_header( c, { 'block_index' : count, 'species' : spec, 'sequence_index' : spec_counts[ spec ] }, suffix = "%s_%i_%i" % ( spec, count, spec_counts[ spec ] ) ) ) - out.write( "%s\n" % c.text ) - out.write( "\n" ) - out.close() - print "%i MAF blocks converted to FASTA." % ( count ) - - -if __name__ == "__main__": __main__() + spec_counts[ spec ] += 1 + out.write( "%s\n" % maf_utilities.get_fasta_header( c, { 'block_index' : count, 'species' : spec, 'sequence_index' : spec_counts[ spec ] }, suffix = "%s_%i_%i" % ( spec, count, spec_counts[ spec ] ) ) ) + out.write( "%s\n" % c.text ) + out.write( "\n" ) + out.close() + print "%i MAF blocks converted to FASTA." % ( count ) + + +if __name__ == "__main__": __main__() diff --git a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py index 036727b5f98..2ddb00cbe43 100644 --- a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py @@ -1,32 +1,32 @@ -#!/usr/bin/env python -#Dan Blankenberg - -import sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) +#!/usr/bin/env python +#Dan Blankenberg + +import sys +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) import bx.align.maf -from galaxy.tools.util import maf_utilities - -assert sys.version_info[:2] >= ( 2, 4 ) - -def __main__(): - output_name = sys.argv.pop(1) - input_name = sys.argv.pop(1) - species = sys.argv.pop(1) - out = open(output_name,'w') - count = 0 - #write interval header line - out.write( "#chrom\tstart\tend\tstrand\n" ) - try: - for block in bx.align.maf.Reader( open( input_name, 'r' ) ): - for c in maf_utilities.iter_components_by_src_start( block, species ): - if c is not None: - out.write( "%s\t%i\t%i\t%s\n" % ( maf_utilities.src_split( c.src )[-1], c.get_forward_strand_start(), c.get_forward_strand_end(), c.strand ) ) - count += 1 - except Exception, e: - print >> sys.stderr, "There was a problem processing your input: %s" % e - out.close() - print "%i MAF blocks converted to Genomic Intervals for species %s." % ( count, species ) - - -if __name__ == "__main__": __main__() +from galaxy.tools.util import maf_utilities + +assert sys.version_info[:2] >= ( 2, 4 ) + +def __main__(): + output_name = sys.argv.pop(1) + input_name = sys.argv.pop(1) + species = sys.argv.pop(1) + out = open(output_name,'w') + count = 0 + #write interval header line + out.write( "#chrom\tstart\tend\tstrand\n" ) + try: + for block in bx.align.maf.Reader( open( input_name, 'r' ) ): + for c in maf_utilities.iter_components_by_src_start( block, species ): + if c is not None: + out.write( "%s\t%i\t%i\t%s\n" % ( maf_utilities.src_split( c.src )[-1], c.get_forward_strand_start(), c.get_forward_strand_end(), c.strand ) ) + count += 1 + except Exception, e: + print >> sys.stderr, "There was a problem processing your input: %s" % e + out.close() + print "%i MAF blocks converted to Genomic Intervals for species %s." % ( count, species ) + + +if __name__ == "__main__": __main__() diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py index 7fea37cb034..817fd9f7fd3 100644 --- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.py @@ -21,7 +21,7 @@ galhtmlprefix = """
""" -plinke = 'plink' +plinke = 'plink' def timenow(): @@ -51,7 +51,7 @@ def pruneLD(plinktasks=[],cd='./',vclbase = []): except: alog.append('### %s Strange - no std out from plink when running command line\n%s\n' % (timenow(),' '.join(vcl))) return alog - + def makeLDreduced(basename,infpath=None,outfpath=None,plinke='plink',forcerebuild=False,returnFname=False, winsize="60", winmove="40", r2thresh="0.1" ): @@ -79,11 +79,11 @@ def main(): need to work with rgenetics composite datatypes so in and out are html files with data in extrafiles path - .. raw:: xml + .. raw:: xml - pbed_ldreduced_converter.py '$input1.extra_files_path/$input1.metadata.base_name' '$winsize' '$winmove' '$r2thresh' - '$output1' '$output1.files_path' 'plink' + pbed_ldreduced_converter.py '$input1.extra_files_path/$input1.metadata.base_name' '$winsize' '$winmove' '$r2thresh' + '$output1' '$output1.files_path' 'plink' """ @@ -116,7 +116,7 @@ def main(): f.write('
  • %s
  • \n' % (os.path.split(data)[-1],os.path.split(data)[-1])) f.write("
    ") f.close() - + if __name__ == "__main__": main() diff --git a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py index 74d160079af..31edb23f572 100644 --- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py +++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py @@ -1,80 +1,80 @@ -# for rgenetics - lped to pbed -# where to stop with converters -# pbed might be central -# eg lped/eigen/fbat/snpmatrix all to pbed -# and pbed to lped/eigen/fbat/snpmatrix ? -# that's a lot of converters -import sys,os,time,subprocess - - -prog = os.path.split(sys.argv[0])[-1] -myversion = 'Oct 10 2009' - -galhtmlprefix = """ - - - - - - - - - -
    -""" - -def timenow(): - """return current time as a string - """ - return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) +# for rgenetics - lped to pbed +# where to stop with converters +# pbed might be central +# eg lped/eigen/fbat/snpmatrix all to pbed +# and pbed to lped/eigen/fbat/snpmatrix ? +# that's a lot of converters +import sys,os,time,subprocess + + +prog = os.path.split(sys.argv[0])[-1] +myversion = 'Oct 10 2009' + +galhtmlprefix = """ + + + + + + + + + +
    +""" + +def timenow(): + """return current time as a string + """ + return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) + - def rgConv(inpedfilepath,outhtmlname,outfilepath,plink): """ """ - - basename = os.path.split(inpedfilepath)[-1] # get basename + + basename = os.path.split(inpedfilepath)[-1] # get basename outroot = os.path.join(outfilepath,basename) cl = '%s --noweb --bfile %s --recode --out %s ' % (plink,inpedfilepath,outroot) p = subprocess.Popen(cl,shell=True,cwd=outfilepath) retval = p.wait() # run plink - - - -def main(): - """ - need to work with rgenetics composite datatypes - so in and out are html files with data in extrafiles path - pbed_to_lped_converter.py '$input1/$input1.metadata.base_name' - '$output1' '$output1.extra_files_path' '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink' - - """ - nparm = 4 - if len(sys.argv) < nparm: - sys.stderr.write('## %s called with %s - needs %d parameters \n' % (myname,sys.argv,nparm)) - sys.exit(1) - inpedfilepath = sys.argv[1] - outhtmlname = sys.argv[2] - outfilepath = sys.argv[3] - try: - os.makedirs(outfilepath) - except: - pass - plink = sys.argv[4] - rgConv(inpedfilepath,outhtmlname,outfilepath,plink) - f = file(outhtmlname,'w') - f.write(galhtmlprefix % prog) - flist = os.listdir(outfilepath) - s = '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info - print s - f.write('
    %s\n
      ' % (s)) - for i, data in enumerate( flist ): - f.write('
    1. %s
    2. \n' % (os.path.split(data)[-1],os.path.split(data)[-1])) - f.write("
    ") - f.close() - - - -if __name__ == "__main__": - main() + + + +def main(): + """ + need to work with rgenetics composite datatypes + so in and out are html files with data in extrafiles path + pbed_to_lped_converter.py '$input1/$input1.metadata.base_name' + '$output1' '$output1.extra_files_path' '${GALAXY_DATA_INDEX_DIR}/rg/bin/plink' + + """ + nparm = 4 + if len(sys.argv) < nparm: + sys.stderr.write('## %s called with %s - needs %d parameters \n' % (myname,sys.argv,nparm)) + sys.exit(1) + inpedfilepath = sys.argv[1] + outhtmlname = sys.argv[2] + outfilepath = sys.argv[3] + try: + os.makedirs(outfilepath) + except: + pass + plink = sys.argv[4] + rgConv(inpedfilepath,outhtmlname,outfilepath,plink) + f = file(outhtmlname,'w') + f.write(galhtmlprefix % prog) + flist = os.listdir(outfilepath) + s = '## Rgenetics: http://rgenetics.org Galaxy Tools %s %s' % (prog,timenow()) # becomes info + print s + f.write('
    %s\n
      ' % (s)) + for i, data in enumerate( flist ): + f.write('
    1. %s
    2. \n' % (os.path.split(data)[-1],os.path.split(data)[-1])) + f.write("
    ") + f.close() + + + +if __name__ == "__main__": + main() diff --git a/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py index 63e7c5ce5a4..803a2cb26a7 100644 --- a/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py +++ b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.py @@ -17,7 +17,7 @@ def __main__(): for i, line in enumerate( open( input_name ) ): complete_interval = False line = line.rstrip( '\r\n' ) - if line: + if line: if line.startswith( HEADER_STARTS_WITH ): header_lines += 1 else: diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py index d26e28f70df..76a1e2a07ac 100644 --- a/lib/galaxy/datatypes/converters/sam_to_bam.py +++ b/lib/galaxy/datatypes/converters/sam_to_bam.py @@ -19,12 +19,12 @@ def __main__(): #Parse Command Line parser = optparse.OptionParser() (options, args) = parser.parse_args() - + assert len( args ) == 2, 'You must specify the input and output filenames' input_filename, output_filename = args - + tmp_dir = tempfile.mkdtemp( prefix='tmp-sam_to_bam_converter-' ) - + #convert to SAM unsorted_bam_filename = os.path.join( tmp_dir, 'unsorted.bam' ) unsorted_stderr_filename = os.path.join( tmp_dir, 'unsorted.stderr' ) @@ -43,14 +43,14 @@ def __main__(): else: break stderr.close() - + #sort sam, so indexing will not fail sorted_stderr_filename = os.path.join( tmp_dir, 'sorted.stderr' ) sorting_prefix = os.path.join( tmp_dir, 'sorted_bam' ) cmd = 'samtools sort -o "%s" "%s" > "%s"' % ( unsorted_bam_filename, sorting_prefix, output_filename ) proc = subprocess.Popen( args=cmd, stderr=open( sorted_stderr_filename, 'wb' ), shell=True, cwd=tmp_dir ) return_code = proc.wait() - + if return_code: stderr_target = sys.stderr else: @@ -63,7 +63,7 @@ def __main__(): else: break stderr.close() - + cleanup_before_exit( tmp_dir ) if __name__=="__main__": __main__() diff --git a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py index 40f6b93250c..a55b7d2b5e4 100644 --- a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py @@ -16,20 +16,19 @@ def main(): # Read options, args. parser = optparse.OptionParser() (options, args) = parser.parse_args() - in_file, out_file = args - + in_file, out_file = args + # Do conversion. index = Indexes() - reader = galaxy_utils.sequence.vcf.Reader( open( in_file ) ) + reader = galaxy_utils.sequence.vcf.Reader( open( in_file ) ) offset = reader.metadata_len for vcf_line in reader: - # VCF format provides a chrom and 1-based position for each variant. + # VCF format provides a chrom and 1-based position for each variant. # IntervalIndex expects 0-based coordinates. index.add( vcf_line.chrom, vcf_line.pos-1, vcf_line.pos, offset ) offset += len( vcf_line.raw_line ) - + index.write( open( out_file, "w" ) ) -if __name__ == "__main__": +if __name__ == "__main__": main() - \ No newline at end of file diff --git a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py index df29dede1d6..d439aa7fc51 100644 --- a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py +++ b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ -Uses pysam to bgzip a vcf file as-is. +Uses pysam to bgzip a vcf file as-is. Headers, which are important, are kept. Original ordering, which may be specifically needed by tools or external display applications, is also maintained. @@ -17,8 +17,8 @@ def main(): parser = optparse.OptionParser() (options, args) = parser.parse_args() input_fname, output_fname = args - + ctabix.tabix_compress(input_fname, output_fname, force=True) - -if __name__ == "__main__": + +if __name__ == "__main__": main() diff --git a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py index 7b4e3b862dd..de13538cdcb 100644 --- a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py @@ -11,19 +11,19 @@ from bx.arrays.wiggle import WiggleReader BLOCK_SIZE = 100 def main(): - + input_fname = sys.argv[1] out_fname = sys.argv[2] - + reader = WiggleReader( open( input_fname ) ) - + # Fill array from reader d = array_tree_dict_from_reader( reader, {}, block_size = BLOCK_SIZE ) - + for array_tree in d.itervalues(): array_tree.root.build_summary() - + FileArrayTreeDict.dict_to_file( d, open( out_fname, "w" ) ) -if __name__ == "__main__": +if __name__ == "__main__": main() \ No newline at end of file diff --git a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py index 5943be5cce5..c70d2c9f459 100644 --- a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py +++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py @@ -17,16 +17,16 @@ def stop_err( msg ): sys.exit() def main(): - if len( sys.argv ) > 1: + if len( sys.argv ) > 1: in_file = open( sys.argv[1] ) - else: + else: in_file = open( sys.stdin ) - + if len( sys.argv ) > 2: out_file = open( sys.argv[2], "w" ) else: out_file = sys.stdout - + try: for fields in bx.wiggle.IntervalReader( UCSCOutWrapper( in_file ) ): out_file.write( "%s\n" % "\t".join( map( str, fields ) ) ) diff --git a/lib/galaxy/datatypes/coverage.py b/lib/galaxy/datatypes/coverage.py index 24da96a6b56..e5f78ba9b59 100644 --- a/lib/galaxy/datatypes/coverage.py +++ b/lib/galaxy/datatypes/coverage.py @@ -15,7 +15,7 @@ log = logging.getLogger(__name__) class LastzCoverage( Tabular ): file_ext = "coverage" - + MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter ) MetadataElement( name="positionCol", default=2, desc="Position column", param=metadata.ColumnParameter ) MetadataElement( name="forwardCol", default=3, desc="Forward or aggregate read column", param=metadata.ColumnParameter ) @@ -44,7 +44,7 @@ class LastzCoverage( Tabular ): t_end = math.ceil( end / resolution ) x = numpy.arange( t_start, t_end ) * resolution y = data[ t_start : t_end ] - + return zip(x.tolist(), y.tolist()) def get_track_resolution( self, dataset, start, end): diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index dd1e89464b4..fc2f3fc6495 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -282,14 +282,14 @@ class Data( object ): tmpfh = open( tmpf ) # CANNOT clean up - unlink/rmdir was always failing because file handle retained to return - must rely on a cron job to clean up tmp trans.response.set_content_type( "application/x-zip-compressed" ) - trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.zip"' % outfname + trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.zip"' % outfname return tmpfh else: trans.response.set_content_type( "application/x-tar" ) outext = 'tgz' if params.do_action == 'tbz': outext = 'tbz' - trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.%s"' % (outfname,outext) + trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.%s"' % (outfname,outext) archive.wsgi_status = trans.response.wsgi_status() archive.wsgi_headeritems = trans.response.wsgi_headeritems() return archive.stream diff --git a/lib/galaxy/datatypes/dataproviders/base.py b/lib/galaxy/datatypes/dataproviders/base.py index 0878133eb60..bb3ca8998a8 100644 --- a/lib/galaxy/datatypes/dataproviders/base.py +++ b/lib/galaxy/datatypes/dataproviders/base.py @@ -304,7 +304,7 @@ class MultiSourceDataProvider( DataProvider ): self.source = self.validate_source( source ) except exceptions.InvalidDataProviderSource, invalid_source: continue - + parent_gen = super( MultiSourceDataProvider, self ).__iter__() for datum in parent_gen: yield datum diff --git a/lib/galaxy/datatypes/dataproviders/line.py b/lib/galaxy/datatypes/dataproviders/line.py index 12945946bb9..4d2e127b0d2 100644 --- a/lib/galaxy/datatypes/dataproviders/line.py +++ b/lib/galaxy/datatypes/dataproviders/line.py @@ -262,7 +262,7 @@ class BlockDataProvider( base.LimitedOffsetDataProvider ): """ if self.limit != None and self.num_data_returned >= self.limit: return None - + last_block = self.assemble_current_block() self.num_data_read += 1 diff --git a/lib/galaxy/datatypes/display_applications/application.py b/lib/galaxy/datatypes/display_applications/application.py index 70eef7918e9..a2baeaf4fd3 100644 --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -128,7 +128,7 @@ class PopulatedDisplayApplicationLink( object ): self.data = data self.dataset_hash = dataset_hash self.user_hash = user_hash - self.trans = trans + self.trans = trans self.ready, self.parameters = self.link.build_parameter_dict( self.data, self.dataset_hash, self.user_hash, trans, app_kwds ) def display_ready( self ): return self.ready diff --git a/lib/galaxy/datatypes/display_applications/parameters.py b/lib/galaxy/datatypes/display_applications/parameters.py index efdb1240027..b4d382a626c 100644 --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -10,9 +10,9 @@ DEFAULT_DATASET_NAME = 'dataset' class DisplayApplicationParameter( object ): """ Abstract Class for Display Application Parameters """ - + type = None - + @classmethod def from_elem( cls, elem, link ): param_type = elem.get( 'type', None ) @@ -42,9 +42,9 @@ class DisplayApplicationParameter( object ): class DisplayApplicationDataParameter( DisplayApplicationParameter ): """ Parameter that returns a file_name containing the requested content """ - + type = 'data' - + def __init__( self, elem, link ): DisplayApplicationParameter.__init__( self, elem, link ) self.extensions = elem.get( 'format', None ) @@ -113,7 +113,7 @@ class DisplayApplicationDataParameter( DisplayApplicationParameter ): return False def ready( self, other_values ): value = self._get_dataset_like_object( other_values ) - if value: + if value: if value.state == value.states.OK: return True elif value.state == value.states.ERROR: @@ -122,9 +122,9 @@ class DisplayApplicationDataParameter( DisplayApplicationParameter ): class DisplayApplicationTemplateParameter( DisplayApplicationParameter ): """ Parameter that returns a string containing the requested content """ - + type = 'template' - + def __init__( self, elem, link ): DisplayApplicationParameter.__init__( self, elem, link ) self.text = elem.text or '' @@ -154,7 +154,7 @@ class DisplayParameterValueWrapper( object ): if self.parameter.guess_mime_type: mime, encoding = mimetypes.guess_type( self._url ) if not mime: - mime = self.trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( self._url )[ -1 ], None ) + mime = self.trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( self._url )[ -1 ], None ) if mime: return mime return 'text/plain' @@ -193,7 +193,7 @@ class DisplayDataValueWrapper( DisplayParameterValueWrapper ): if self.parameter.guess_mime_type: mime, encoding = mimetypes.guess_type( self._url ) if not mime: - mime = self.trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( self._url )[ -1 ], None ) + mime = self.trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( self._url )[ -1 ], None ) if mime: return mime if hasattr( self.value, 'get_mime' ): diff --git a/lib/galaxy/datatypes/display_applications/util.py b/lib/galaxy/datatypes/display_applications/util.py index a045fc32a3f..916312fe218 100644 --- a/lib/galaxy/datatypes/display_applications/util.py +++ b/lib/galaxy/datatypes/display_applications/util.py @@ -10,7 +10,7 @@ def encode_dataset_user( trans, dataset, user ): user_hash = 'None' else: user_hash = str( user.id ) - # Pad to a multiple of 8 with leading "!" + # Pad to a multiple of 8 with leading "!" user_hash = ( "!" * ( 8 - len( user_hash ) % 8 ) ) + user_hash cipher = Blowfish.new( str( dataset.create_time ) ) user_hash = cipher.encrypt( user_hash ).encode( 'hex' ) diff --git a/lib/galaxy/datatypes/genetics.py b/lib/galaxy/datatypes/genetics.py index 011d0abf261..f9e4eb57eed 100644 --- a/lib/galaxy/datatypes/genetics.py +++ b/lib/galaxy/datatypes/genetics.py @@ -1,811 +1,811 @@ -""" -rgenetics datatypes -Use at your peril -Ross Lazarus -for the rgenetics and galaxy projects - -genome graphs datatypes derived from Interval datatypes -genome graphs datasets have a header row with appropriate columnames -The first column is always the marker - eg columname = rs, first row= rs12345 if the rows are snps -subsequent row values are all numeric ! Will fail if any non numeric (eg '+' or 'NA') values -ross lazarus for rgenetics -august 20 2007 -""" - -import logging, os, sys, time, tempfile, shutil, string, glob -import data -from galaxy import util -from cgi import escape -import urllib, binascii -from galaxy.web import url_for -from galaxy.datatypes import metadata -from galaxy.datatypes.metadata import MetadataElement -from galaxy.datatypes.data import Text -from galaxy.datatypes.tabular import Tabular -from galaxy.datatypes.images import Html -from galaxy.datatypes.interval import Interval -from galaxy.util.hash_util import * - -gal_Log = logging.getLogger(__name__) -verbose = False - -class GenomeGraphs( Tabular ): - """ - Tab delimited data containing a marker id and any number of numeric values - """ - - MetadataElement( name="markerCol", default=1, desc="Marker ID column", param=metadata.ColumnParameter ) - MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True ) - MetadataElement( name="column_types", default=[], desc="Column types", readonly=True, visible=False ) - file_ext = 'gg' - - def __init__(self, **kwd): - """ - Initialize gg datatype, by adding UCSC display apps - """ - Tabular.__init__(self, **kwd) - self.add_display_app ( 'ucsc', 'Genome Graph', 'as_ucsc_display_file', 'ucsc_links' ) - - - def set_meta(self,dataset,**kwd): - Tabular.set_meta( self, dataset, **kwd) - dataset.metadata.markerCol = 1 - header = file(dataset.file_name,'r').readlines()[0].strip().split('\t') - dataset.metadata.columns = len(header) - t = ['numeric' for x in header] - t[0] = 'string' - dataset.metadata.column_types = t - return True - - def as_ucsc_display_file( self, dataset, **kwd ): - """ - Returns file - """ - return file(dataset.file_name,'r') - - def ucsc_links( self, dataset, type, app, base_url ): - """ - from the ever-helpful angie hinrichs angie@soe.ucsc.edu - a genome graphs call looks like this - - http://genome.ucsc.edu/cgi-bin/hgGenome?clade=mammal&org=Human&db=hg18&hgGenome_dataSetName=dname - &hgGenome_dataSetDescription=test&hgGenome_formatType=best%20guess&hgGenome_markerType=best%20guess - &hgGenome_columnLabels=best%20guess&hgGenome_maxVal=&hgGenome_labelVals= - &hgGenome_maxGapToFill=25000000&hgGenome_uploadFile=http://galaxy.esphealth.org/datasets/333/display/index - &hgGenome_doSubmitUpload=submit - - Galaxy gives this for an interval file - - http://genome.ucsc.edu/cgi-bin/hgTracks?db=hg18&position=chr1:1-1000&hgt.customText= - http%3A%2F%2Fgalaxy.esphealth.org%2Fdisplay_as%3Fid%3D339%26display_app%3Ducsc - - """ - ret_val = [] - ggtail = 'hgGenome_doSubmitUpload=submit' - if not dataset.dbkey: - dataset.dbkey = 'hg18' # punt! - if dataset.has_data(): - for site_name, site_url in util.get_ucsc_by_build(dataset.dbkey): - if site_name in app.config.ucsc_display_sites: - site_url = site_url.replace('/hgTracks?','/hgGenome?') # for genome graphs - internal_url = "%s" % url_for( controller='dataset', - dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name ) - display_url = "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, url_for( controller='root' ), dataset.id, type) - display_url = urllib.quote_plus( display_url ) - # was display_url = urllib.quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) ) - #redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) ) - sl = ["%sdb=%s" % (site_url,dataset.dbkey ),] - #sl.append("&hgt.customText=%s") - sl.append("&hgGenome_dataSetName=%s&hgGenome_dataSetDescription=%s" % (dataset.name, 'GalaxyGG_data')) - sl.append("&hgGenome_formatType=best guess&hgGenome_markerType=best guess") - sl.append("&hgGenome_columnLabels=first row&hgGenome_maxVal=&hgGenome_labelVals=") - sl.append("&hgGenome_doSubmitUpload=submit") - sl.append("&hgGenome_maxGapToFill=25000000&hgGenome_uploadFile=%s" % display_url) - s = ''.join(sl) - s = urllib.quote_plus(s) - redirect_url = s - link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url ) - ret_val.append( (site_name, link) ) - return ret_val - - def make_html_table( self, dataset, skipchars=[] ): - """ - Create HTML table, used for displaying peek - """ - npeek = 5 - out = [''] - f = open(dataset.file_name,'r') - d = f.readlines()[:5] - if len(d) == 0: - out = "Cannot find anything to parse in %s" % dataset.name - return out - hasheader = 0 - try: - test = ['%f' % x for x in d[0][1:]] # first is name - see if starts all numerics - except: - hasheader = 1 - try: - # Generate column header - out.append( '' ) - if hasheader: - for i, name in enumerate(d[0].split() ): - out.append( '' % ( str( i+1 ), name ) ) - d.pop(0) - out.append('') - for row in d: - out.append('') - out.append(''.join(['' % x for x in row.split()])) - out.append('') - out.append( '
    %s.%s
    %s
    ' ) - out = "".join( out ) - except Exception, exc: - out = "Can't create peek %s" % exc - return out - - def validate( self, dataset ): - """ - Validate a gg file - all numeric after header row - """ - errors = list() - infile = open(dataset.file_name, "r") - header= infile.next() # header - for i,row in enumerate(infile): - ll = row.strip().split('\t')[1:] # first is alpha feature identifier - badvals = [] - for j,x in enumerate(ll): - try: - x = float(x) - except: - badval.append('col%d:%s' % (j+1,x)) - if len(badvals) > 0: - errors.append('row %d, %s' % (' '.join(badvals))) - return errors - - def sniff( self, filename ): - """ - Determines whether the file is in gg format - """ - f = open(filename,'r') - headers = f.readline().split() - rows = [f.readline().split()[1:] for x in range(3)] # small sample - #headers = get_headers( filename, '\t' ) - for row in rows: - try: - nums = [float(x) for x in row] # first col has been removed - except: - return false - return true - - def get_mime(self): - """Returns the mime type of the datatype""" - return 'application/vnd.ms-excel' - - -class rgTabList(Tabular): - """ - for sampleid and for featureid lists of exclusions or inclusions in the clean tool - featureid subsets on statistical criteria -> specialized display such as gg - """ - file_ext = "rgTList" - - - def __init__(self, **kwd): - """ - Initialize featurelistt datatype - """ - Tabular.__init__( self, **kwd ) - self.column_names = [] - - def display_peek( self, dataset ): - """Returns formated html of peek""" - return Tabular.make_html_table( self, dataset, column_names=self.column_names ) - - def get_mime(self): - """Returns the mime type of the datatype""" - return 'text/html' - - -class rgSampleList(rgTabList): - """ - for sampleid exclusions or inclusions in the clean tool - output from QC eg excess het, gender error, ibd pair member,eigen outlier,excess mendel errors,... - since they can be uploaded, should be flexible - but they are persistent at least - same infrastructure for expression? - """ - file_ext = "rgSList" - - def __init__(self, **kwd): - """ - Initialize samplelist datatype - """ - rgTabList.__init__( self, **kwd ) - self.column_names[0] = 'FID' - self.column_names[1] = 'IID' - # this is what Plink wants as at 2009 - - def sniff(self,filename): - infile = open(dataset.file_name, "r") - header= infile.next() # header - if header[0] == 'FID' and header[1] == 'IID': - return True - else: - return False - -class rgFeatureList( rgTabList ): - """ - for featureid lists of exclusions or inclusions in the clean tool - output from QC eg low maf, high missingness, bad hwe in controls, excess mendel errors,... - featureid subsets on statistical criteria -> specialized display such as gg - same infrastructure for expression? - """ - file_ext = "rgFList" - - def __init__(self, **kwd): - """Initialize featurelist datatype""" - rgTabList.__init__( self, **kwd ) - for i,s in enumerate(['#FeatureId', 'Chr', 'Genpos', 'Mappos']): - self.column_names[i] = s - - -class Rgenetics(Html): - """ - base class to use for rgenetics datatypes - derived from html - composite datatype elements - stored in extra files path - """ - - MetadataElement( name="base_name", desc="base name for all transformed versions of this genetic dataset", default='RgeneticsData', - readonly=True, set_in_upload=True) - - composite_type = 'auto_primary_file' - allow_datatype_change = False - file_ext = 'rgenetics' - - def generate_primary_file( self, dataset = None ): - rval = ['Rgenetics Galaxy Composite Dataset

    '] - rval.append('

    This composite dataset is composed of the following files:

      ') - for composite_name, composite_file in self.get_composite_files( dataset = dataset ).iteritems(): - fn = composite_name - opt_text = '' - if composite_file.optional: - opt_text = ' (optional)' - if composite_file.get('description'): - rval.append( '
    • %s (%s)%s
    • ' % ( fn, fn, composite_file.get('description'), opt_text ) ) - else: - rval.append( '
    • %s%s
    • ' % ( fn, fn, opt_text ) ) - rval.append( '
    ' ) - return "\n".join( rval ) - - def regenerate_primary_file(self,dataset): - """ - cannot do this until we are setting metadata - """ - bn = dataset.metadata.base_name - efp = dataset.extra_files_path - flist = os.listdir(efp) - rval = ['Files for Composite Dataset %s

    Composite %s contains:

      ' % (dataset.name,dataset.name)] - for i,fname in enumerate(flist): - sfname = os.path.split(fname)[-1] - f,e = os.path.splitext(fname) - rval.append( '
    • %s
    • ' % ( sfname, sfname) ) - rval.append( '
    ' ) - f = file(dataset.file_name,'w') - f.write("\n".join( rval )) - f.write('\n') - f.close() - - def get_mime(self): - """Returns the mime type of the datatype""" - return 'text/html' - - - def set_meta( self, dataset, **kwd ): - - """ - for lped/pbed eg - - """ - Html.set_meta( self, dataset, **kwd ) - if kwd.get('overwrite') == False: - if verbose: - gal_Log.debug('@@@ rgenetics set_meta called with overwrite = False') - return True - try: - efp = dataset.extra_files_path - except: - if verbose: - gal_Log.debug('@@@rgenetics set_meta failed %s - dataset %s has no efp ?' % (sys.exc_info()[0], dataset.name)) - return False - try: - flist = os.listdir(efp) - except: - if verbose: gal_Log.debug('@@@rgenetics set_meta failed %s - dataset %s has no efp ?' % (sys.exc_info()[0],dataset.name)) - return False - if len(flist) == 0: - if verbose: - gal_Log.debug('@@@rgenetics set_meta failed - %s efp %s is empty?' % (dataset.name,efp)) - return False - self.regenerate_primary_file(dataset) - if not dataset.info: - dataset.info = 'Galaxy genotype datatype object' - if not dataset.blurb: - dataset.blurb = 'Composite file - Rgenetics Galaxy toolkit' - return True - - - -class SNPMatrix(Rgenetics): - """ - BioC SNPMatrix Rgenetics data collections - """ - file_ext="snpmatrix" - - def set_peek( self, dataset, **kwd ): - if not dataset.dataset.purged: - dataset.peek = "Binary RGenetics file" - dataset.blurb = data.nice_size( dataset.get_size() ) - else: - dataset.peek = 'file does not exist' - dataset.blurb = 'file purged from disk' - - def sniff(self,filename): - """ need to check the file header hex code - """ - infile = open(dataset.file_name, "b") - head = infile.read(16) - head = [hex(x) for x in head] - if head <> '': - return False - else: - return True - - -class Lped(Rgenetics): - """ - linkage pedigree (ped,map) Rgenetics data collections - """ - file_ext="lped" - - def __init__( self, **kwd ): - Rgenetics.__init__(self, **kwd) - self.add_composite_file( '%s.ped', description = 'Pedigree File', substitute_name_with_metadata = 'base_name', is_binary = False ) - self.add_composite_file( '%s.map', description = 'Map File', substitute_name_with_metadata = 'base_name', is_binary = False ) - - -class Pphe(Rgenetics): - """ - Plink phenotype file - header must have FID\tIID... Rgenetics data collections - """ - file_ext="pphe" - - def __init__( self, **kwd ): - Rgenetics.__init__(self, **kwd) - self.add_composite_file( '%s.pphe', description = 'Plink Phenotype File', substitute_name_with_metadata = 'base_name', is_binary = False ) - - - - -class Fphe(Rgenetics): - """ - fbat pedigree file - mad format with ! as first char on header row - Rgenetics data collections - """ - file_ext="fphe" - - def __init__( self, **kwd ): - Rgenetics.__init__(self, **kwd) - self.add_composite_file( '%s.fphe', description = 'FBAT Phenotype File', substitute_name_with_metadata = 'base_name' ) - -class Phe(Rgenetics): - """ - Phenotype file - """ - file_ext="phe" - - def __init__( self, **kwd ): - Rgenetics.__init__(self, **kwd) - self.add_composite_file( '%s.phe', description = 'Phenotype File', substitute_name_with_metadata = 'base_name', - is_binary = False ) - - - -class Fped(Rgenetics): - """ - FBAT pedigree format - single file, map is header row of rs numbers. Strange. - Rgenetics data collections - """ - file_ext="fped" - - def __init__( self, **kwd ): - Rgenetics.__init__(self, **kwd) - self.add_composite_file( '%s.fped', description = 'FBAT format pedfile', substitute_name_with_metadata = 'base_name', - is_binary = False ) - - -class Pbed(Rgenetics): - """ - Plink Binary compressed 2bit/geno Rgenetics data collections - """ - file_ext="pbed" - - def __init__( self, **kwd ): - Rgenetics.__init__(self, **kwd) - self.add_composite_file( '%s.bim', substitute_name_with_metadata = 'base_name', is_binary = False ) - self.add_composite_file( '%s.bed', substitute_name_with_metadata = 'base_name', is_binary = True ) - self.add_composite_file( '%s.fam', substitute_name_with_metadata = 'base_name', is_binary = False ) - -class ldIndep(Rgenetics): - """ - LD (a good measure of redundancy of information) depleted Plink Binary compressed 2bit/geno - This is really a plink binary, but some tools work better with less redundancy so are constrained to - these files - """ - file_ext="ldreduced" - - def __init__( self, **kwd ): - Rgenetics.__init__(self, **kwd) - self.add_composite_file( '%s.bim', substitute_name_with_metadata = 'base_name', is_binary = False ) - self.add_composite_file( '%s.bed', substitute_name_with_metadata = 'base_name', is_binary = True ) - self.add_composite_file( '%s.fam', substitute_name_with_metadata = 'base_name', is_binary = False ) - - -class Eigenstratgeno(Rgenetics): - """ - Eigenstrat format - may be able to get rid of this - if we move to shellfish - Rgenetics data collections - """ - file_ext="eigenstratgeno" - - def __init__( self, **kwd ): - Rgenetics.__init__(self, **kwd) - self.add_composite_file( '%s.eigenstratgeno', substitute_name_with_metadata = 'base_name', is_binary = False ) - self.add_composite_file( '%s.ind', substitute_name_with_metadata = 'base_name', is_binary = False ) - self.add_composite_file( '%s.map', substitute_name_with_metadata = 'base_name', is_binary = False ) - - - -class Eigenstratpca(Rgenetics): - """ - Eigenstrat PCA file for case control adjustment - Rgenetics data collections - """ - file_ext="eigenstratpca" - - def __init__( self, **kwd ): - Rgenetics.__init__(self, **kwd) - self.add_composite_file( '%s.eigenstratpca', description = 'Eigenstrat PCA file', substitute_name_with_metadata = 'base_name' ) - - -class Snptest(Rgenetics): - """ - BioC snptest Rgenetics data collections - """ - file_ext="snptest" - - -class Pheno(Tabular): - """ - base class for pheno files - """ - file_ext = 'pheno' - - -class RexpBase( Html ): - """ - base class for BioC data structures in Galaxy - must be constructed with the pheno data in place since that - goes into the metadata for each instance - """ - MetadataElement( name="columns", default=0, desc="Number of columns", visible=True ) - MetadataElement( name="column_names", default=[], desc="Column names", visible=True ) - MetadataElement(name="pheCols",default=[],desc="Select list for potentially interesting variables",visible=True) - MetadataElement( name="base_name", - desc="base name for all transformed versions of this expression dataset", default='rexpression', set_in_upload=True) - MetadataElement( name="pheno_path", desc="Path to phenotype data for this experiment", default="rexpression.pheno", visible=True) - file_ext = 'rexpbase' - html_table = None - is_binary = True - composite_type = 'auto_primary_file' - allow_datatype_change = False - - - def __init__( self, **kwd ): - Html.__init__(self,**kwd) - self.add_composite_file( '%s.pheno', description = 'Phenodata tab text file', - substitute_name_with_metadata = 'base_name', is_binary=False) - - def generate_primary_file( self, dataset = None ): - """ - This is called only at upload to write the html file - cannot rename the datasets here - they come with the default unfortunately - """ - return 'AutoGenerated Primary File for Composite Dataset' - - def get_mime(self): - """Returns the mime type of the datatype""" - return 'text/html' - - def get_phecols(self, phenolist=[], maxConc=20): - """ - sept 2009: cannot use whitespace to split - make a more complex structure here - and adjust the methods that rely on this structure - return interesting phenotype column names for an rexpression eset or affybatch - to use in array subsetting and so on. Returns a data structure for a - dynamic Galaxy select parameter. - A column with only 1 value doesn't change, so is not interesting for - analysis. A column with a different value in every row is equivalent to a unique - identifier so is also not interesting for anova or limma analysis - both these - are removed after the concordance (count of unique terms) is constructed for each - column. Then a complication - each remaining pair of columns is tested for - redundancy - if two columns are always paired, then only one is needed :) - """ - for nrows,row in enumerate(phenolist): # construct concordance - if len(row.strip()) == 0: - break - row = row.strip().split('\t') - if nrows == 0: # set up from header - head = row - totcols = len(row) - concordance = [{} for x in head] # list of dicts - else: - for col,code in enumerate(row): # keep column order correct - if col >= totcols: - gal_Log.warning('### get_phecols error in pheno file - row %d col %d (%s) longer than header %s' % (nrows, col, row, head)) - else: - concordance[col].setdefault(code,0) # first one is zero - concordance[col][code] += 1 - useCols = [] - useConc = [] # columns of interest to keep - nrows = len(phenolist) - nrows -= 1 # drop head from count - for c,conc in enumerate(concordance): # c is column number - if (len(conc) > 1) and (len(conc) < min(nrows,maxConc)): # not all same and not all different!! - useConc.append(conc) # keep concordance - useCols.append(c) # keep column - nuse = len(useCols) - # now to check for pairs of concordant columns - drop one of these. - delme = [] - p = phenolist[1:] # drop header - plist = [x.strip().split('\t') for x in p] # list of lists - phe = [[x[i] for i in useCols] for x in plist if len(x) >= totcols] # strip unused data - for i in range(0,(nuse-1)): # for each interesting column - for j in range(i+1,nuse): - kdict = {} - for row in phe: # row is a list of lists - k = '%s%s' % (row[i],row[j]) # composite key - kdict[k] = k - if (len(kdict.keys()) == len(concordance[useCols[j]])): # i and j are always matched - delme.append(j) - delme = list(set(delme)) # remove dupes - listCol = [] - delme.sort() - delme.reverse() # must delete from far end! - for i in delme: - del useConc[i] # get rid of concordance - del useCols[i] # and usecols entry - for i,conc in enumerate(useConc): # these are all unique columns for the design matrix - ccounts = [(conc.get(code,0),code) for code in conc.keys()] # decorate - ccounts.sort() - cc = [(x[1],x[0]) for x in ccounts] # list of code count tuples - codeDetails = (head[useCols[i]],cc) # ('foo',[('a',3),('b',11),..]) - listCol.append(codeDetails) - if len(listCol) > 0: - res = listCol - # metadata.pheCols becomes [('bar;22,zot;113','foo'), ...] - else: - res = [('no usable phenotype columns found',[('?',0),]),] - return res - - - - def get_pheno(self,dataset): - """ - expects a .pheno file in the extra_files_dir - ugh - note that R is wierd and adds the row.name in - the header so the columns are all wrong - unless you tell it not to. - A file can be written as - write.table(file='foo.pheno',pData(foo),sep='\t',quote=F,row.names=F) - """ - p = file(dataset.metadata.pheno_path,'r').readlines() - if len(p) > 0: # should only need to fix an R pheno file once - head = p[0].strip().split('\t') - line1 = p[1].strip().split('\t') - if len(head) < len(line1): - head.insert(0,'ChipFileName') # fix R write.table b0rken-ness - p[0] = '\t'.join(head) - else: - p = [] - return '\n'.join(p) - - def set_peek( self, dataset, **kwd ): - """ - expects a .pheno file in the extra_files_dir - ugh - note that R is weird and does not include the row.name in - the header. why?""" - if not dataset.dataset.purged: - pp = os.path.join(dataset.extra_files_path,'%s.pheno' % dataset.metadata.base_name) - try: - p = file(pp,'r').readlines() - except: - p = ['##failed to find %s' % pp,] - dataset.peek = ''.join(p[:5]) - dataset.blurb = 'Galaxy Rexpression composite file' - else: - dataset.peek = 'file does not exist\n' - dataset.blurb = 'file purged from disk' - - def get_peek( self, dataset ): - """ - expects a .pheno file in the extra_files_dir - ugh - """ - pp = os.path.join(dataset.extra_files_path,'%s.pheno' % dataset.metadata.base_name) - try: - p = file(pp,'r').readlines() - except: - p = ['##failed to find %s' % pp] - return ''.join(p[:5]) - - def get_file_peek(self,filename): - """ - can't really peek at a filename - need the extra_files_path and such? - """ - h = '## rexpression get_file_peek: no file found' - try: - h = file(filename,'r').readlines() - except: - pass - return ''.join(h[:5]) - - def regenerate_primary_file(self,dataset): - """ - cannot do this until we are setting metadata - """ - bn = dataset.metadata.base_name - flist = os.listdir(dataset.extra_files_path) - rval = ['Files for Composite Dataset %s

    Comprises the following files:

      ' % (bn)] - for i,fname in enumerate(flist): - sfname = os.path.split(fname)[-1] - rval.append( '
    • %s' % ( sfname, sfname ) ) - rval.append( '
    ' ) - f = file(dataset.file_name,'w') - f.write("\n".join( rval )) - f.write('\n') - f.close() - - def init_meta( self, dataset, copy_from=None ): - if copy_from: - dataset.metadata = copy_from.metadata - - def set_meta( self, dataset, **kwd ): - - """ - NOTE we apply the tabular machinary to the phenodata extracted - from a BioC eSet or affybatch. - - """ - Html.set_meta(self, dataset, **kwd) - try: - flist = os.listdir(dataset.extra_files_path) - except: - if verbose: - gal_Log.debug('@@@rexpression set_meta failed - no dataset?') - return False - bn = dataset.metadata.base_name - if not bn: - for f in flist: - n = os.path.splitext(f)[0] - bn = n - dataset.metadata.base_name = bn - if not bn: - bn = '?' - dataset.metadata.base_name = bn - pn = '%s.pheno' % (bn) - pp = os.path.join(dataset.extra_files_path,pn) - dataset.metadata.pheno_path=pp - try: - pf = file(pp,'r').readlines() # read the basename.phenodata in the extra_files_path - except: - pf = None - if pf: - h = pf[0].strip() - h = h.split('\t') # hope is header - h = [escape(x) for x in h] - dataset.metadata.column_names = h - dataset.metadata.columns = len(h) - dataset.peek = ''.join(pf[:5]) - else: - dataset.metadata.column_names = [] - dataset.metadata.columns = 0 - dataset.peek = 'No pheno file found' - if pf and len(pf) > 1: - dataset.metadata.pheCols = self.get_phecols(phenolist=pf) - else: - dataset.metadata.pheCols = [('','No useable phenotypes found',False),] - #self.regenerate_primary_file(dataset) - if not dataset.info: - dataset.info = 'Galaxy Expression datatype object' - if not dataset.blurb: - dataset.blurb = 'R loadable BioC expression object for the Rexpression Galaxy toolkit' - return True - - def make_html_table( self, pp='nothing supplied from peek\n'): - """ - Create HTML table, used for displaying peek - """ - out = ['',] - p = pp.split('\n') - try: - # Generate column header - for i,row in enumerate(p): - lrow = row.strip().split('\t') - if i == 0: - orow = ['' % escape(x) for x in lrow] - orow.insert(0,'') - orow.append('') - else: - orow = ['' % escape(x) for x in lrow] - orow.insert(0,'') - orow.append('') - out.append(''.join(orow)) - out.append( '
    %s
    %s
    ' ) - out = "\n".join( out ) - except Exception, exc: - out = "Can't create html table %s" % str( exc ) - return out - - def display_peek( self, dataset ): - """ - Returns formatted html of peek - """ - out=self.make_html_table(dataset.peek) - return out - - def get_mime(self): - """ - Returns the mime type of the datatype - """ - return 'text/html' - - -class Affybatch( RexpBase ): - """ - derived class for BioC data structures in Galaxy - """ - - file_ext = "affybatch" - - def __init__( self, **kwd ): - RexpBase.__init__(self, **kwd) - self.add_composite_file( '%s.affybatch', description = 'AffyBatch R object saved to file', - substitute_name_with_metadata = 'base_name', is_binary=True ) - -class Eset( RexpBase ): - """ - derived class for BioC data structures in Galaxy - """ - file_ext = "eset" - - def __init__( self, **kwd ): - RexpBase.__init__(self, **kwd) - self.add_composite_file( '%s.eset', description = 'ESet R object saved to file', - substitute_name_with_metadata = 'base_name', is_binary = True ) - - -class MAlist( RexpBase ): - """ - derived class for BioC data structures in Galaxy - """ - file_ext = "malist" - - def __init__( self, **kwd ): - RexpBase.__init__(self, **kwd) - self.add_composite_file( '%s.malist', description = 'MAlist R object saved to file', - substitute_name_with_metadata = 'base_name', is_binary = True ) - - -if __name__ == '__main__': - import doctest, sys - doctest.testmod(sys.modules[__name__]) - +""" +rgenetics datatypes +Use at your peril +Ross Lazarus +for the rgenetics and galaxy projects + +genome graphs datatypes derived from Interval datatypes +genome graphs datasets have a header row with appropriate columnames +The first column is always the marker - eg columname = rs, first row= rs12345 if the rows are snps +subsequent row values are all numeric ! Will fail if any non numeric (eg '+' or 'NA') values +ross lazarus for rgenetics +august 20 2007 +""" + +import logging, os, sys, time, tempfile, shutil, string, glob +import data +from galaxy import util +from cgi import escape +import urllib, binascii +from galaxy.web import url_for +from galaxy.datatypes import metadata +from galaxy.datatypes.metadata import MetadataElement +from galaxy.datatypes.data import Text +from galaxy.datatypes.tabular import Tabular +from galaxy.datatypes.images import Html +from galaxy.datatypes.interval import Interval +from galaxy.util.hash_util import * + +gal_Log = logging.getLogger(__name__) +verbose = False + +class GenomeGraphs( Tabular ): + """ + Tab delimited data containing a marker id and any number of numeric values + """ + + MetadataElement( name="markerCol", default=1, desc="Marker ID column", param=metadata.ColumnParameter ) + MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True ) + MetadataElement( name="column_types", default=[], desc="Column types", readonly=True, visible=False ) + file_ext = 'gg' + + def __init__(self, **kwd): + """ + Initialize gg datatype, by adding UCSC display apps + """ + Tabular.__init__(self, **kwd) + self.add_display_app ( 'ucsc', 'Genome Graph', 'as_ucsc_display_file', 'ucsc_links' ) + + + def set_meta(self,dataset,**kwd): + Tabular.set_meta( self, dataset, **kwd) + dataset.metadata.markerCol = 1 + header = file(dataset.file_name,'r').readlines()[0].strip().split('\t') + dataset.metadata.columns = len(header) + t = ['numeric' for x in header] + t[0] = 'string' + dataset.metadata.column_types = t + return True + + def as_ucsc_display_file( self, dataset, **kwd ): + """ + Returns file + """ + return file(dataset.file_name,'r') + + def ucsc_links( self, dataset, type, app, base_url ): + """ + from the ever-helpful angie hinrichs angie@soe.ucsc.edu + a genome graphs call looks like this + + http://genome.ucsc.edu/cgi-bin/hgGenome?clade=mammal&org=Human&db=hg18&hgGenome_dataSetName=dname + &hgGenome_dataSetDescription=test&hgGenome_formatType=best%20guess&hgGenome_markerType=best%20guess + &hgGenome_columnLabels=best%20guess&hgGenome_maxVal=&hgGenome_labelVals= + &hgGenome_maxGapToFill=25000000&hgGenome_uploadFile=http://galaxy.esphealth.org/datasets/333/display/index + &hgGenome_doSubmitUpload=submit + + Galaxy gives this for an interval file + + http://genome.ucsc.edu/cgi-bin/hgTracks?db=hg18&position=chr1:1-1000&hgt.customText= + http%3A%2F%2Fgalaxy.esphealth.org%2Fdisplay_as%3Fid%3D339%26display_app%3Ducsc + + """ + ret_val = [] + ggtail = 'hgGenome_doSubmitUpload=submit' + if not dataset.dbkey: + dataset.dbkey = 'hg18' # punt! + if dataset.has_data(): + for site_name, site_url in util.get_ucsc_by_build(dataset.dbkey): + if site_name in app.config.ucsc_display_sites: + site_url = site_url.replace('/hgTracks?','/hgGenome?') # for genome graphs + internal_url = "%s" % url_for( controller='dataset', + dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name ) + display_url = "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, url_for( controller='root' ), dataset.id, type) + display_url = urllib.quote_plus( display_url ) + # was display_url = urllib.quote_plus( "%s/display_as?id=%i&display_app=%s" % (base_url, dataset.id, type) ) + #redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop) ) + sl = ["%sdb=%s" % (site_url,dataset.dbkey ),] + #sl.append("&hgt.customText=%s") + sl.append("&hgGenome_dataSetName=%s&hgGenome_dataSetDescription=%s" % (dataset.name, 'GalaxyGG_data')) + sl.append("&hgGenome_formatType=best guess&hgGenome_markerType=best guess") + sl.append("&hgGenome_columnLabels=first row&hgGenome_maxVal=&hgGenome_labelVals=") + sl.append("&hgGenome_doSubmitUpload=submit") + sl.append("&hgGenome_maxGapToFill=25000000&hgGenome_uploadFile=%s" % display_url) + s = ''.join(sl) + s = urllib.quote_plus(s) + redirect_url = s + link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url ) + ret_val.append( (site_name, link) ) + return ret_val + + def make_html_table( self, dataset, skipchars=[] ): + """ + Create HTML table, used for displaying peek + """ + npeek = 5 + out = [''] + f = open(dataset.file_name,'r') + d = f.readlines()[:5] + if len(d) == 0: + out = "Cannot find anything to parse in %s" % dataset.name + return out + hasheader = 0 + try: + test = ['%f' % x for x in d[0][1:]] # first is name - see if starts all numerics + except: + hasheader = 1 + try: + # Generate column header + out.append( '' ) + if hasheader: + for i, name in enumerate(d[0].split() ): + out.append( '' % ( str( i+1 ), name ) ) + d.pop(0) + out.append('') + for row in d: + out.append('') + out.append(''.join(['' % x for x in row.split()])) + out.append('') + out.append( '
    %s.%s
    %s
    ' ) + out = "".join( out ) + except Exception, exc: + out = "Can't create peek %s" % exc + return out + + def validate( self, dataset ): + """ + Validate a gg file - all numeric after header row + """ + errors = list() + infile = open(dataset.file_name, "r") + header= infile.next() # header + for i,row in enumerate(infile): + ll = row.strip().split('\t')[1:] # first is alpha feature identifier + badvals = [] + for j,x in enumerate(ll): + try: + x = float(x) + except: + badval.append('col%d:%s' % (j+1,x)) + if len(badvals) > 0: + errors.append('row %d, %s' % (' '.join(badvals))) + return errors + + def sniff( self, filename ): + """ + Determines whether the file is in gg format + """ + f = open(filename,'r') + headers = f.readline().split() + rows = [f.readline().split()[1:] for x in range(3)] # small sample + #headers = get_headers( filename, '\t' ) + for row in rows: + try: + nums = [float(x) for x in row] # first col has been removed + except: + return false + return true + + def get_mime(self): + """Returns the mime type of the datatype""" + return 'application/vnd.ms-excel' + + +class rgTabList(Tabular): + """ + for sampleid and for featureid lists of exclusions or inclusions in the clean tool + featureid subsets on statistical criteria -> specialized display such as gg + """ + file_ext = "rgTList" + + + def __init__(self, **kwd): + """ + Initialize featurelistt datatype + """ + Tabular.__init__( self, **kwd ) + self.column_names = [] + + def display_peek( self, dataset ): + """Returns formated html of peek""" + return Tabular.make_html_table( self, dataset, column_names=self.column_names ) + + def get_mime(self): + """Returns the mime type of the datatype""" + return 'text/html' + + +class rgSampleList(rgTabList): + """ + for sampleid exclusions or inclusions in the clean tool + output from QC eg excess het, gender error, ibd pair member,eigen outlier,excess mendel errors,... + since they can be uploaded, should be flexible + but they are persistent at least + same infrastructure for expression? + """ + file_ext = "rgSList" + + def __init__(self, **kwd): + """ + Initialize samplelist datatype + """ + rgTabList.__init__( self, **kwd ) + self.column_names[0] = 'FID' + self.column_names[1] = 'IID' + # this is what Plink wants as at 2009 + + def sniff(self,filename): + infile = open(dataset.file_name, "r") + header= infile.next() # header + if header[0] == 'FID' and header[1] == 'IID': + return True + else: + return False + +class rgFeatureList( rgTabList ): + """ + for featureid lists of exclusions or inclusions in the clean tool + output from QC eg low maf, high missingness, bad hwe in controls, excess mendel errors,... + featureid subsets on statistical criteria -> specialized display such as gg + same infrastructure for expression? + """ + file_ext = "rgFList" + + def __init__(self, **kwd): + """Initialize featurelist datatype""" + rgTabList.__init__( self, **kwd ) + for i,s in enumerate(['#FeatureId', 'Chr', 'Genpos', 'Mappos']): + self.column_names[i] = s + + +class Rgenetics(Html): + """ + base class to use for rgenetics datatypes + derived from html - composite datatype elements + stored in extra files path + """ + + MetadataElement( name="base_name", desc="base name for all transformed versions of this genetic dataset", default='RgeneticsData', + readonly=True, set_in_upload=True) + + composite_type = 'auto_primary_file' + allow_datatype_change = False + file_ext = 'rgenetics' + + def generate_primary_file( self, dataset = None ): + rval = ['Rgenetics Galaxy Composite Dataset

    '] + rval.append('

    This composite dataset is composed of the following files:

      ') + for composite_name, composite_file in self.get_composite_files( dataset = dataset ).iteritems(): + fn = composite_name + opt_text = '' + if composite_file.optional: + opt_text = ' (optional)' + if composite_file.get('description'): + rval.append( '
    • %s (%s)%s
    • ' % ( fn, fn, composite_file.get('description'), opt_text ) ) + else: + rval.append( '
    • %s%s
    • ' % ( fn, fn, opt_text ) ) + rval.append( '
    ' ) + return "\n".join( rval ) + + def regenerate_primary_file(self,dataset): + """ + cannot do this until we are setting metadata + """ + bn = dataset.metadata.base_name + efp = dataset.extra_files_path + flist = os.listdir(efp) + rval = ['Files for Composite Dataset %s

    Composite %s contains:

      ' % (dataset.name,dataset.name)] + for i,fname in enumerate(flist): + sfname = os.path.split(fname)[-1] + f,e = os.path.splitext(fname) + rval.append( '
    • %s
    • ' % ( sfname, sfname) ) + rval.append( '
    ' ) + f = file(dataset.file_name,'w') + f.write("\n".join( rval )) + f.write('\n') + f.close() + + def get_mime(self): + """Returns the mime type of the datatype""" + return 'text/html' + + + def set_meta( self, dataset, **kwd ): + + """ + for lped/pbed eg + + """ + Html.set_meta( self, dataset, **kwd ) + if kwd.get('overwrite') == False: + if verbose: + gal_Log.debug('@@@ rgenetics set_meta called with overwrite = False') + return True + try: + efp = dataset.extra_files_path + except: + if verbose: + gal_Log.debug('@@@rgenetics set_meta failed %s - dataset %s has no efp ?' % (sys.exc_info()[0], dataset.name)) + return False + try: + flist = os.listdir(efp) + except: + if verbose: gal_Log.debug('@@@rgenetics set_meta failed %s - dataset %s has no efp ?' % (sys.exc_info()[0],dataset.name)) + return False + if len(flist) == 0: + if verbose: + gal_Log.debug('@@@rgenetics set_meta failed - %s efp %s is empty?' % (dataset.name,efp)) + return False + self.regenerate_primary_file(dataset) + if not dataset.info: + dataset.info = 'Galaxy genotype datatype object' + if not dataset.blurb: + dataset.blurb = 'Composite file - Rgenetics Galaxy toolkit' + return True + + + +class SNPMatrix(Rgenetics): + """ + BioC SNPMatrix Rgenetics data collections + """ + file_ext="snpmatrix" + + def set_peek( self, dataset, **kwd ): + if not dataset.dataset.purged: + dataset.peek = "Binary RGenetics file" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def sniff(self,filename): + """ need to check the file header hex code + """ + infile = open(dataset.file_name, "b") + head = infile.read(16) + head = [hex(x) for x in head] + if head <> '': + return False + else: + return True + + +class Lped(Rgenetics): + """ + linkage pedigree (ped,map) Rgenetics data collections + """ + file_ext="lped" + + def __init__( self, **kwd ): + Rgenetics.__init__(self, **kwd) + self.add_composite_file( '%s.ped', description = 'Pedigree File', substitute_name_with_metadata = 'base_name', is_binary = False ) + self.add_composite_file( '%s.map', description = 'Map File', substitute_name_with_metadata = 'base_name', is_binary = False ) + + +class Pphe(Rgenetics): + """ + Plink phenotype file - header must have FID\tIID... Rgenetics data collections + """ + file_ext="pphe" + + def __init__( self, **kwd ): + Rgenetics.__init__(self, **kwd) + self.add_composite_file( '%s.pphe', description = 'Plink Phenotype File', substitute_name_with_metadata = 'base_name', is_binary = False ) + + + + +class Fphe(Rgenetics): + """ + fbat pedigree file - mad format with ! as first char on header row + Rgenetics data collections + """ + file_ext="fphe" + + def __init__( self, **kwd ): + Rgenetics.__init__(self, **kwd) + self.add_composite_file( '%s.fphe', description = 'FBAT Phenotype File', substitute_name_with_metadata = 'base_name' ) + +class Phe(Rgenetics): + """ + Phenotype file + """ + file_ext="phe" + + def __init__( self, **kwd ): + Rgenetics.__init__(self, **kwd) + self.add_composite_file( '%s.phe', description = 'Phenotype File', substitute_name_with_metadata = 'base_name', + is_binary = False ) + + + +class Fped(Rgenetics): + """ + FBAT pedigree format - single file, map is header row of rs numbers. Strange. + Rgenetics data collections + """ + file_ext="fped" + + def __init__( self, **kwd ): + Rgenetics.__init__(self, **kwd) + self.add_composite_file( '%s.fped', description = 'FBAT format pedfile', substitute_name_with_metadata = 'base_name', + is_binary = False ) + + +class Pbed(Rgenetics): + """ + Plink Binary compressed 2bit/geno Rgenetics data collections + """ + file_ext="pbed" + + def __init__( self, **kwd ): + Rgenetics.__init__(self, **kwd) + self.add_composite_file( '%s.bim', substitute_name_with_metadata = 'base_name', is_binary = False ) + self.add_composite_file( '%s.bed', substitute_name_with_metadata = 'base_name', is_binary = True ) + self.add_composite_file( '%s.fam', substitute_name_with_metadata = 'base_name', is_binary = False ) + +class ldIndep(Rgenetics): + """ + LD (a good measure of redundancy of information) depleted Plink Binary compressed 2bit/geno + This is really a plink binary, but some tools work better with less redundancy so are constrained to + these files + """ + file_ext="ldreduced" + + def __init__( self, **kwd ): + Rgenetics.__init__(self, **kwd) + self.add_composite_file( '%s.bim', substitute_name_with_metadata = 'base_name', is_binary = False ) + self.add_composite_file( '%s.bed', substitute_name_with_metadata = 'base_name', is_binary = True ) + self.add_composite_file( '%s.fam', substitute_name_with_metadata = 'base_name', is_binary = False ) + + +class Eigenstratgeno(Rgenetics): + """ + Eigenstrat format - may be able to get rid of this + if we move to shellfish + Rgenetics data collections + """ + file_ext="eigenstratgeno" + + def __init__( self, **kwd ): + Rgenetics.__init__(self, **kwd) + self.add_composite_file( '%s.eigenstratgeno', substitute_name_with_metadata = 'base_name', is_binary = False ) + self.add_composite_file( '%s.ind', substitute_name_with_metadata = 'base_name', is_binary = False ) + self.add_composite_file( '%s.map', substitute_name_with_metadata = 'base_name', is_binary = False ) + + + +class Eigenstratpca(Rgenetics): + """ + Eigenstrat PCA file for case control adjustment + Rgenetics data collections + """ + file_ext="eigenstratpca" + + def __init__( self, **kwd ): + Rgenetics.__init__(self, **kwd) + self.add_composite_file( '%s.eigenstratpca', description = 'Eigenstrat PCA file', substitute_name_with_metadata = 'base_name' ) + + +class Snptest(Rgenetics): + """ + BioC snptest Rgenetics data collections + """ + file_ext="snptest" + + +class Pheno(Tabular): + """ + base class for pheno files + """ + file_ext = 'pheno' + + +class RexpBase( Html ): + """ + base class for BioC data structures in Galaxy + must be constructed with the pheno data in place since that + goes into the metadata for each instance + """ + MetadataElement( name="columns", default=0, desc="Number of columns", visible=True ) + MetadataElement( name="column_names", default=[], desc="Column names", visible=True ) + MetadataElement(name="pheCols",default=[],desc="Select list for potentially interesting variables",visible=True) + MetadataElement( name="base_name", + desc="base name for all transformed versions of this expression dataset", default='rexpression', set_in_upload=True) + MetadataElement( name="pheno_path", desc="Path to phenotype data for this experiment", default="rexpression.pheno", visible=True) + file_ext = 'rexpbase' + html_table = None + is_binary = True + composite_type = 'auto_primary_file' + allow_datatype_change = False + + + def __init__( self, **kwd ): + Html.__init__(self,**kwd) + self.add_composite_file( '%s.pheno', description = 'Phenodata tab text file', + substitute_name_with_metadata = 'base_name', is_binary=False) + + def generate_primary_file( self, dataset = None ): + """ + This is called only at upload to write the html file + cannot rename the datasets here - they come with the default unfortunately + """ + return 'AutoGenerated Primary File for Composite Dataset' + + def get_mime(self): + """Returns the mime type of the datatype""" + return 'text/html' + + def get_phecols(self, phenolist=[], maxConc=20): + """ + sept 2009: cannot use whitespace to split - make a more complex structure here + and adjust the methods that rely on this structure + return interesting phenotype column names for an rexpression eset or affybatch + to use in array subsetting and so on. Returns a data structure for a + dynamic Galaxy select parameter. + A column with only 1 value doesn't change, so is not interesting for + analysis. A column with a different value in every row is equivalent to a unique + identifier so is also not interesting for anova or limma analysis - both these + are removed after the concordance (count of unique terms) is constructed for each + column. Then a complication - each remaining pair of columns is tested for + redundancy - if two columns are always paired, then only one is needed :) + """ + for nrows,row in enumerate(phenolist): # construct concordance + if len(row.strip()) == 0: + break + row = row.strip().split('\t') + if nrows == 0: # set up from header + head = row + totcols = len(row) + concordance = [{} for x in head] # list of dicts + else: + for col,code in enumerate(row): # keep column order correct + if col >= totcols: + gal_Log.warning('### get_phecols error in pheno file - row %d col %d (%s) longer than header %s' % (nrows, col, row, head)) + else: + concordance[col].setdefault(code,0) # first one is zero + concordance[col][code] += 1 + useCols = [] + useConc = [] # columns of interest to keep + nrows = len(phenolist) + nrows -= 1 # drop head from count + for c,conc in enumerate(concordance): # c is column number + if (len(conc) > 1) and (len(conc) < min(nrows,maxConc)): # not all same and not all different!! + useConc.append(conc) # keep concordance + useCols.append(c) # keep column + nuse = len(useCols) + # now to check for pairs of concordant columns - drop one of these. + delme = [] + p = phenolist[1:] # drop header + plist = [x.strip().split('\t') for x in p] # list of lists + phe = [[x[i] for i in useCols] for x in plist if len(x) >= totcols] # strip unused data + for i in range(0,(nuse-1)): # for each interesting column + for j in range(i+1,nuse): + kdict = {} + for row in phe: # row is a list of lists + k = '%s%s' % (row[i],row[j]) # composite key + kdict[k] = k + if (len(kdict.keys()) == len(concordance[useCols[j]])): # i and j are always matched + delme.append(j) + delme = list(set(delme)) # remove dupes + listCol = [] + delme.sort() + delme.reverse() # must delete from far end! + for i in delme: + del useConc[i] # get rid of concordance + del useCols[i] # and usecols entry + for i,conc in enumerate(useConc): # these are all unique columns for the design matrix + ccounts = [(conc.get(code,0),code) for code in conc.keys()] # decorate + ccounts.sort() + cc = [(x[1],x[0]) for x in ccounts] # list of code count tuples + codeDetails = (head[useCols[i]],cc) # ('foo',[('a',3),('b',11),..]) + listCol.append(codeDetails) + if len(listCol) > 0: + res = listCol + # metadata.pheCols becomes [('bar;22,zot;113','foo'), ...] + else: + res = [('no usable phenotype columns found',[('?',0),]),] + return res + + + + def get_pheno(self,dataset): + """ + expects a .pheno file in the extra_files_dir - ugh + note that R is wierd and adds the row.name in + the header so the columns are all wrong - unless you tell it not to. + A file can be written as + write.table(file='foo.pheno',pData(foo),sep='\t',quote=F,row.names=F) + """ + p = file(dataset.metadata.pheno_path,'r').readlines() + if len(p) > 0: # should only need to fix an R pheno file once + head = p[0].strip().split('\t') + line1 = p[1].strip().split('\t') + if len(head) < len(line1): + head.insert(0,'ChipFileName') # fix R write.table b0rken-ness + p[0] = '\t'.join(head) + else: + p = [] + return '\n'.join(p) + + def set_peek( self, dataset, **kwd ): + """ + expects a .pheno file in the extra_files_dir - ugh + note that R is weird and does not include the row.name in + the header. why?""" + if not dataset.dataset.purged: + pp = os.path.join(dataset.extra_files_path,'%s.pheno' % dataset.metadata.base_name) + try: + p = file(pp,'r').readlines() + except: + p = ['##failed to find %s' % pp,] + dataset.peek = ''.join(p[:5]) + dataset.blurb = 'Galaxy Rexpression composite file' + else: + dataset.peek = 'file does not exist\n' + dataset.blurb = 'file purged from disk' + + def get_peek( self, dataset ): + """ + expects a .pheno file in the extra_files_dir - ugh + """ + pp = os.path.join(dataset.extra_files_path,'%s.pheno' % dataset.metadata.base_name) + try: + p = file(pp,'r').readlines() + except: + p = ['##failed to find %s' % pp] + return ''.join(p[:5]) + + def get_file_peek(self,filename): + """ + can't really peek at a filename - need the extra_files_path and such? + """ + h = '## rexpression get_file_peek: no file found' + try: + h = file(filename,'r').readlines() + except: + pass + return ''.join(h[:5]) + + def regenerate_primary_file(self,dataset): + """ + cannot do this until we are setting metadata + """ + bn = dataset.metadata.base_name + flist = os.listdir(dataset.extra_files_path) + rval = ['Files for Composite Dataset %s

    Comprises the following files:

      ' % (bn)] + for i,fname in enumerate(flist): + sfname = os.path.split(fname)[-1] + rval.append( '
    • %s' % ( sfname, sfname ) ) + rval.append( '
    ' ) + f = file(dataset.file_name,'w') + f.write("\n".join( rval )) + f.write('\n') + f.close() + + def init_meta( self, dataset, copy_from=None ): + if copy_from: + dataset.metadata = copy_from.metadata + + def set_meta( self, dataset, **kwd ): + + """ + NOTE we apply the tabular machinary to the phenodata extracted + from a BioC eSet or affybatch. + + """ + Html.set_meta(self, dataset, **kwd) + try: + flist = os.listdir(dataset.extra_files_path) + except: + if verbose: + gal_Log.debug('@@@rexpression set_meta failed - no dataset?') + return False + bn = dataset.metadata.base_name + if not bn: + for f in flist: + n = os.path.splitext(f)[0] + bn = n + dataset.metadata.base_name = bn + if not bn: + bn = '?' + dataset.metadata.base_name = bn + pn = '%s.pheno' % (bn) + pp = os.path.join(dataset.extra_files_path,pn) + dataset.metadata.pheno_path=pp + try: + pf = file(pp,'r').readlines() # read the basename.phenodata in the extra_files_path + except: + pf = None + if pf: + h = pf[0].strip() + h = h.split('\t') # hope is header + h = [escape(x) for x in h] + dataset.metadata.column_names = h + dataset.metadata.columns = len(h) + dataset.peek = ''.join(pf[:5]) + else: + dataset.metadata.column_names = [] + dataset.metadata.columns = 0 + dataset.peek = 'No pheno file found' + if pf and len(pf) > 1: + dataset.metadata.pheCols = self.get_phecols(phenolist=pf) + else: + dataset.metadata.pheCols = [('','No useable phenotypes found',False),] + #self.regenerate_primary_file(dataset) + if not dataset.info: + dataset.info = 'Galaxy Expression datatype object' + if not dataset.blurb: + dataset.blurb = 'R loadable BioC expression object for the Rexpression Galaxy toolkit' + return True + + def make_html_table( self, pp='nothing supplied from peek\n'): + """ + Create HTML table, used for displaying peek + """ + out = ['',] + p = pp.split('\n') + try: + # Generate column header + for i,row in enumerate(p): + lrow = row.strip().split('\t') + if i == 0: + orow = ['' % escape(x) for x in lrow] + orow.insert(0,'') + orow.append('') + else: + orow = ['' % escape(x) for x in lrow] + orow.insert(0,'') + orow.append('') + out.append(''.join(orow)) + out.append( '
    %s
    %s
    ' ) + out = "\n".join( out ) + except Exception, exc: + out = "Can't create html table %s" % str( exc ) + return out + + def display_peek( self, dataset ): + """ + Returns formatted html of peek + """ + out=self.make_html_table(dataset.peek) + return out + + def get_mime(self): + """ + Returns the mime type of the datatype + """ + return 'text/html' + + +class Affybatch( RexpBase ): + """ + derived class for BioC data structures in Galaxy + """ + + file_ext = "affybatch" + + def __init__( self, **kwd ): + RexpBase.__init__(self, **kwd) + self.add_composite_file( '%s.affybatch', description = 'AffyBatch R object saved to file', + substitute_name_with_metadata = 'base_name', is_binary=True ) + +class Eset( RexpBase ): + """ + derived class for BioC data structures in Galaxy + """ + file_ext = "eset" + + def __init__( self, **kwd ): + RexpBase.__init__(self, **kwd) + self.add_composite_file( '%s.eset', description = 'ESet R object saved to file', + substitute_name_with_metadata = 'base_name', is_binary = True ) + + +class MAlist( RexpBase ): + """ + derived class for BioC data structures in Galaxy + """ + file_ext = "malist" + + def __init__( self, **kwd ): + RexpBase.__init__(self, **kwd) + self.add_composite_file( '%s.malist', description = 'MAlist R object saved to file', + substitute_name_with_metadata = 'base_name', is_binary = True ) + + +if __name__ == '__main__': + import doctest, sys + doctest.testmod(sys.modules[__name__]) + diff --git a/lib/galaxy/datatypes/images.py b/lib/galaxy/datatypes/images.py index 6d5436b420d..f2fc663a9f8 100644 --- a/lib/galaxy/datatypes/images.py +++ b/lib/galaxy/datatypes/images.py @@ -26,7 +26,7 @@ log = logging.getLogger(__name__) # TODO: Uploading image files of various types is supported in Galaxy, but on # the main public instance, the display_in_upload is not set for these data # types in datatypes_conf.xml because we do not allow image files to be uploaded -# there. There is currently no API feature that allows uploading files outside +# there. There is currently no API feature that allows uploading files outside # of a data library ( where it requires either the upload_paths or upload_directory # option to be enabled, which is not the case on the main public instance ). Because # of this, we're currently safe, but when the api is enhanced to allow other uploads, @@ -112,7 +112,7 @@ class Pcd( Image ): def sniff(self, filename, image=None): """Determine if the file is in pcd format.""" - return check_image_type( filename, ['PCD'], image ) + return check_image_type( filename, ['PCD'], image ) class Pcx( Image ): @@ -128,7 +128,7 @@ class Ppm( Image ): def sniff(self, filename, image=None): """Determine if the file is in ppm format.""" - return check_image_type( filename, ['PPM'], image ) + return check_image_type( filename, ['PPM'], image ) class Psd( Image ): @@ -136,7 +136,7 @@ class Psd( Image ): def sniff(self, filename, image=None): """Determine if the file is in psd format.""" - return check_image_type( filename, ['PSD'], image ) + return check_image_type( filename, ['PSD'], image ) class Xbm( Image ): @@ -144,7 +144,7 @@ class Xbm( Image ): def sniff(self, filename, image=None): """Determine if the file is in XBM format.""" - return check_image_type( filename, ['XBM'], image ) + return check_image_type( filename, ['XBM'], image ) class Xpm( Image ): @@ -152,7 +152,7 @@ class Xpm( Image ): def sniff(self, filename, image=None): """Determine if the file is in XPM format.""" - return check_image_type( filename, ['XPM'], image ) + return check_image_type( filename, ['XPM'], image ) class Rgb( Image ): @@ -184,7 +184,7 @@ class Eps( Image ): def sniff(self, filename, image=None): """Determine if the file is in eps format.""" - return check_image_type( filename, ['EPS'], image ) + return check_image_type( filename, ['EPS'], image ) class Rast( Image ): @@ -214,7 +214,7 @@ Binary.register_sniffable_binary_format("pdf", "pdf", Pdf) def create_applet_tag_peek( class_name, archive, params ): text = """ - """ % ( class_name, archive ) @@ -222,13 +222,13 @@ def create_applet_tag_peek( class_name, archive, params ): text += """""" % ( name, value ) text += """ - """ % ( class_name, archive ) for name, value in params.iteritems(): text += """""" % ( name, value ) - text += """ + text += """ @@ -248,7 +248,7 @@ class Gmaj( data.Data ): "nobutton": "false", "urlpause" :"100", "debug": "false", - "posturl": "history_add_to?%s" % "&".join( map( lambda x: "%s=%s" % ( x[0], quote_plus( str( x[1] ) ) ), [ ( 'copy_access_from', dataset.id), ( 'history_id', dataset.history_id ), ( 'ext', 'maf' ), ( 'name', 'GMAJ Output on data %s' % dataset.hid ), ( 'info', 'Added by GMAJ' ), ( 'dbkey', dataset.dbkey ) ] ) ) + "posturl": "history_add_to?%s" % "&".join( map( lambda x: "%s=%s" % ( x[0], quote_plus( str( x[1] ) ) ), [ ( 'copy_access_from', dataset.id), ( 'history_id', dataset.history_id ), ( 'ext', 'maf' ), ( 'name', 'GMAJ Output on data %s' % dataset.hid ), ( 'info', 'Added by GMAJ' ), ( 'dbkey', dataset.dbkey ) ] ) ) } class_name = "edu.psu.bx.gmaj.MajApplet.class" archive = "/static/gmaj/gmaj.jar" @@ -270,7 +270,7 @@ class Gmaj( data.Data ): return 'application/zip' def sniff(self, filename): """ - NOTE: the sniff.convert_newlines() call in the upload utility will keep Gmaj data types from being + NOTE: the sniff.convert_newlines() call in the upload utility will keep Gmaj data types from being correctly sniffed, but the files can be uploaded (they'll be sniffed as 'txt'). This sniff function is here to provide an example of a sniffer for a zip file. """ @@ -286,7 +286,7 @@ class Gmaj( data.Data ): if not contains_gmaj_file: return False return True - + class Html( data.Text ): """Class describing an html file""" file_ext = "html" diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index b3c097906a1..3d3928f6fe6 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -23,10 +23,10 @@ log = logging.getLogger(__name__) # Contains the meta columns and the words that map to it; list aliases on the # right side of the : in decreasing order of priority -alias_spec = { - 'chromCol' : [ 'chrom' , 'CHROMOSOME' , 'CHROM', 'Chromosome Name' ], +alias_spec = { + 'chromCol' : [ 'chrom' , 'CHROMOSOME' , 'CHROM', 'Chromosome Name' ], 'startCol' : [ 'start' , 'START', 'chromStart', 'txStart', 'Start Position (bp)' ], - 'endCol' : [ 'end' , 'END' , 'STOP', 'chromEnd', 'txEnd', 'End Position (bp)' ], + 'endCol' : [ 'end' , 'END' , 'STOP', 'chromEnd', 'txEnd', 'End Position (bp)' ], 'strandCol' : [ 'strand', 'STRAND', 'Strand' ], 'nameCol' : [ 'name', 'NAME', 'Name', 'name2', 'NAME2', 'Name2', 'Ensembl Gene ID', 'Ensembl Transcript ID', 'Ensembl Peptide ID' ] } @@ -41,7 +41,7 @@ for key, value in alias_spec.items(): # VIEWPORT_MAX_READS_PER_LINE * VIEWPORT_READLINE_BUFFER_SIZE bytes in size, # then we will not generate a viewport for that dataset VIEWPORT_READLINE_BUFFER_SIZE = 1048576 # 1MB -VIEWPORT_MAX_READS_PER_LINE = 10 +VIEWPORT_MAX_READS_PER_LINE = 10 @dataproviders.decorators.has_dataproviders class Interval( Tabular ): @@ -85,7 +85,7 @@ class Interval( Tabular ): setattr( dataset.metadata, meta_name, elems.index( header_val ) + 1 ) break #next meta_name break # Our metadata is set, so break out of the outer loop - else: + else: # Header lines in Interval files are optional. For example, BED is Interval but has no header. # We'll make a best guess at the location of the metadata columns. metadata_is_set = False @@ -114,7 +114,7 @@ class Interval( Tabular ): # int( elems[3] ) # except: # if overwrite or not dataset.metadata.element_is_set( 'nameCol' ): - # dataset.metadata.nameCol = 4 + # dataset.metadata.nameCol = 4 if len( elems ) < 6 or elems[5] not in data.valid_strand: if overwrite or not dataset.metadata.element_is_set( 'strandCol' ): dataset.metadata.strandCol = 0 @@ -154,15 +154,15 @@ class Interval( Tabular ): if end_col is None: end_col = int( dataset.metadata.endCol ) - 1 # Scan lines of file to find a reasonable chromosome and range - chrom = None - start = sys.maxint + chrom = None + start = sys.maxint end = 0 max_col = max( chrom_col, start_col, end_col ) fh = open( dataset.file_name ) while True: line = fh.readline( VIEWPORT_READLINE_BUFFER_SIZE ) # Stop if at end of file - if not line: + if not line: break # Skip comment lines if not line.startswith( '#' ): @@ -173,7 +173,7 @@ class Interval( Tabular ): start = min( start, int( fields[ start_col ] ) ) end = max( end, int( fields[ end_col ] ) ) # Set chrom last, in case start and end are not integers - chrom = fields[ chrom_col ] + chrom = fields[ chrom_col ] viewport_feature_count -= 1 except Exception, e: # Most likely a non-integer field has been encountered @@ -196,7 +196,7 @@ class Interval( Tabular ): except Exception, e: # Unexpected error, possibly missing metadata log.exception( "Exception caught attempting to generate viewport for dataset '%d'", dataset.id ) - return ( None, None, None ) + return ( None, None, None ) def as_ucsc_display_file( self, dataset, **kwd ): """Returns file contents with only the bed data""" @@ -220,7 +220,7 @@ class Interval( Tabular ): else: for elems in util.file_iter(dataset.file_name): tmp = [ elems[c], elems[s], elems[e] ] - os.write(fd, '%s\n' % '\t'.join(tmp) ) + os.write(fd, '%s\n' % '\t'.join(tmp) ) os.close(fd) return open(temp_name) def display_peek( self, dataset ): @@ -233,8 +233,8 @@ class Interval( Tabular ): """ # Filter UCSC sites to only those that are supported by this build and # enabled. - valid_sites = [ ( name, url ) - for name, url in util.get_ucsc_by_build( dataset.dbkey ) + valid_sites = [ ( name, url ) + for name, url in util.get_ucsc_by_build( dataset.dbkey ) if name in app.config.ucsc_display_sites ] if not valid_sites: return [] @@ -246,11 +246,11 @@ class Interval( Tabular ): # Accumulate links for valid sites ret_val = [] for site_name, site_url in valid_sites: - internal_url = url_for( controller='dataset', dataset_id=dataset.id, + internal_url = url_for( controller='dataset', dataset_id=dataset.id, action='display_at', filename='ucsc_' + site_name ) - display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" + display_url = urllib.quote_plus( "%s%s/display_as?id=%i&display_app=%s&authz_method=display_at" % (base_url, url_for( controller='root' ), dataset.id, type) ) - redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" + redirect_url = urllib.quote_plus( "%sdb=%s&position=%s:%s-%s&hgt.customText=%%s" % (site_url, dataset.dbkey, chrom, start, stop ) ) link = '%s?redirect_url=%s&display_url=%s' % ( internal_url, redirect_url, display_url ) ret_val.append( ( site_name, link ) ) @@ -258,7 +258,7 @@ class Interval( Tabular ): def validate( self, dataset ): """Validate an interval file using the bx GenomicIntervalReader""" errors = list() - c, s, e, t = dataset.metadata.chromCol, dataset.metadata.startCol, dataset.metadata.endCol, dataset.metadata.strandCol + c, s, e, t = dataset.metadata.chromCol, dataset.metadata.startCol, dataset.metadata.endCol, dataset.metadata.strandCol c, s, e, t = int(c)-1, int(s)-1, int(e)-1, int(t)-1 infile = open(dataset.file_name, "r") reader = GenomicIntervalReader( @@ -284,10 +284,10 @@ class Interval( Tabular ): def sniff( self, filename ): """ Checks for 'intervalness' - + This format is mostly used by galaxy itself. Valid interval files should include a valid header comment, but this seems to be loosely regulated. - + >>> fname = get_test_fname( 'test_space.txt' ) >>> Interval().sniff( fname ) False @@ -363,14 +363,14 @@ class BedGraph( Interval ): file_ext = "bedgraph" track_type = "LineTrack" data_sources = { "data": "bigwig", "index": "bigwig" } - + def as_ucsc_display_file( self, dataset, **kwd ): """ - Returns file contents as is with no modifications. + Returns file contents as is with no modifications. TODO: this is a functional stub and will need to be enhanced moving forward to provide additional support for bedgraph. """ return open( dataset.file_name ) - + def get_estimated_display_viewport( self, dataset, chrom_col = 0, start_col = 1, end_col = 2 ): """ Set viewport based on dataset's first 100 lines. @@ -417,7 +417,7 @@ class Bed( Interval ): break if metadata_set: break Tabular.set_meta( self, dataset, overwrite = overwrite, skip = i ) - + def as_ucsc_display_file( self, dataset, **kwd ): """Returns file contents with only the bed data. If bed 6+, treat as interval.""" for line in open(dataset.file_name): @@ -440,7 +440,7 @@ class Bed( Interval ): int(fields[9]) if len(fields) > 10: fields2 = fields[10].rstrip(",").split(",") #remove trailing comma and split on comma - for field in fields2: + for field in fields2: int(field) if len(fields) > 11: fields2 = fields[11].rstrip(",").split(",") #remove trailing comma and split on comma @@ -449,23 +449,23 @@ class Bed( Interval ): except: return Interval.as_ucsc_display_file(self, dataset) #only check first line for proper form break - + try: return open(dataset.file_name) except: return "This item contains no content" def sniff( self, filename ): """ Checks for 'bedness' - - BED lines have three required fields and nine additional optional fields. - The number of fields per line must be consistent throughout any single set of data in - an annotation track. The order of the optional fields is binding: lower-numbered + + BED lines have three required fields and nine additional optional fields. + The number of fields per line must be consistent throughout any single set of data in + an annotation track. The order of the optional fields is binding: lower-numbered fields must always be populated if higher-numbered fields are used. The data type of all 12 columns is: 1-str, 2-int, 3-int, 4-str, 5-int, 6-str, 7-int, 8-int, 9-int or list, 10-int, 11-list, 12-list - + For complete details see http://genome.ucsc.edu/FAQ/FAQformat#format1 - + >>> fname = get_test_fname( 'test_tab.bed' ) >>> Bed().sniff( fname ) True @@ -493,7 +493,7 @@ class Bed( Interval ): try: int( hdr[1] ) int( hdr[2] ) - except: + except: return False if len( hdr ) > 4: #hdr[3] is a string, 'name', which defines the name of the BED line - difficult to test for this. @@ -532,7 +532,7 @@ class Bed( Interval ): #hdr[11] is blockStarts - A comma-separated list of block starts. try: block_starts = hdr[11].rstrip(',').split(',') except: return False - if len(block_sizes) != block_count or len(block_starts) != block_count: return False + if len(block_sizes) != block_count or len(block_starts) != block_count: return False else: return False return True except: return False @@ -541,7 +541,7 @@ class BedStrict( Bed ): """Tab delimited data in strict BED format - no non-standard columns allowed""" file_ext = "bedstrict" - + #no user change of datatype allowed allow_datatype_change = False @@ -552,18 +552,18 @@ class BedStrict( Bed ): MetadataElement( name="strandCol", desc="Strand column (click box & select)", readonly=True, param=metadata.MetadataParameter, no_value=0, optional=True ) MetadataElement( name="nameCol", desc="Name/Identifier column (click box & select)", readonly=True, param=metadata.MetadataParameter, no_value=0, optional=True ) MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True, visible=False ) - + def __init__( self, **kwd ): Tabular.__init__( self, **kwd ) self.clear_display_apps() #only new style display applications for this datatype - + def set_meta( self, dataset, overwrite = True, **kwd ): Tabular.set_meta( self, dataset, overwrite = overwrite, **kwd) #need column count first if dataset.metadata.columns >= 4: dataset.metadata.nameCol = 4 if dataset.metadata.columns >= 6: dataset.metadata.strandCol = 6 - + def sniff( self, filename ): return False #NOTE: This would require aggressively validating the entire file @@ -605,8 +605,8 @@ class Gff( Tabular, _RemoteCallMixin ): MetadataElement( name="column_types", default=['str','str','str','int','int','int','str','str','str'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False ) MetadataElement( name="attributes", default=0, desc="Number of attributes", readonly=True, visible=False, no_value=0 ) - MetadataElement( name="attribute_types", default={}, desc="Attribute types", param=metadata.DictParameter, readonly=True, visible=False, no_value=[] ) - + MetadataElement( name="attribute_types", default={}, desc="Attribute types", param=metadata.DictParameter, readonly=True, visible=False, no_value=[] ) + def __init__( self, **kwd ): """Initialize datatype, by adding GBrowse display app""" Tabular.__init__(self, **kwd) @@ -614,11 +614,11 @@ class Gff( Tabular, _RemoteCallMixin ): self.add_display_app( 'gbrowse', 'display in Gbrowse', 'as_gbrowse_display_file', 'gbrowse_links' ) def set_attribute_metadata( self, dataset ): - """ + """ Sets metadata elements for dataset's attributes. """ - # Use first N lines to set metadata for dataset attributes. Attributes + # Use first N lines to set metadata for dataset attributes. Attributes # not found in the first N lines will not have metadata. num_lines = 200 attribute_types = {} @@ -636,7 +636,7 @@ class Gff( Tabular, _RemoteCallMixin ): int( value ) value_type = "int" except: - try: + try: # Try float. float( value ) value_type = "float" @@ -647,7 +647,7 @@ class Gff( Tabular, _RemoteCallMixin ): pass if i + 1 == num_lines: break - + # Set attribute metadata and then set additional metadata. dataset.metadata.attribute_types = attribute_types dataset.metadata.attributes = len( attribute_types ) @@ -681,8 +681,8 @@ class Gff( Tabular, _RemoteCallMixin ): max_line_count = max( viewport_feature_count, 500 ) # maximum number of lines to check; includes comment lines if self.displayable( dataset ): try: - seqid = None - start = sys.maxint + seqid = None + start = sys.maxint stop = 0 fh = open( dataset.file_name ) while True: @@ -693,7 +693,7 @@ class Gff( Tabular, _RemoteCallMixin ): elems = line.rstrip( '\n\r' ).split() if len( elems ) > 3: # line looks like: - # ##sequence-region ctg123 1 1497228 + # ##sequence-region ctg123 1 1497228 seqid = elems[1] # IV start = int( elems[2] )# 6000000 stop = int( elems[3] ) # 6030000 @@ -773,11 +773,11 @@ class Gff( Tabular, _RemoteCallMixin ): def sniff( self, filename ): """ Determines whether the file is in gff format - + GFF lines have nine required fields that must be tab-separated. - + For complete details see http://genome.ucsc.edu/FAQ/FAQformat#format3 - + >>> fname = get_test_fname( 'gff_version_3.gff' ) >>> Gff().sniff( fname ) False @@ -843,10 +843,10 @@ class Gff3( Gff ): valid_gff3_phase = ['.', '0', '1', '2'] column_names = [ 'Seqid', 'Source', 'Type', 'Start', 'End', 'Score', 'Strand', 'Phase', 'Attributes' ] track_type = Interval.track_type - + """Add metadata elements""" MetadataElement( name="column_types", default=['str','str','str','int','int','float','str','int','list'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False ) - + def __init__(self, **kwd): """Initialize datatype, by adding GBrowse display app""" Gff.__init__(self, **kwd) @@ -863,10 +863,10 @@ class Gff3( Gff ): if len( elems ) == 9: try: start = int( elems[3] ) - valid_start = True + valid_start = True except: if elems[3] == '.': - valid_start = True + valid_start = True try: end = int( elems[4] ) valid_end = True @@ -881,10 +881,10 @@ class Gff3( Gff ): def sniff( self, filename ): """ Determines whether the file is in gff version 3 format - + GFF 3 format: - - 1) adds a mechanism for representing more than one level + + 1) adds a mechanism for representing more than one level of hierarchical grouping of features and subfeatures. 2) separates the ideas of group membership and feature name/id 3) constrains the feature type field to be taken from a controlled @@ -893,13 +893,13 @@ class Gff3( Gff ): one group at a time. 5) provides an explicit convention for pairwise alignments 6) provides an explicit convention for features that occupy disjunct regions - + The format consists of 9 columns, separated by tabs (NOT spaces). - + Undefined fields are replaced with the "." character, as described in the original GFF spec. - + For complete details see http://song.sourceforge.net/gff3.shtml - + >>> fname = get_test_fname( 'test.gff' ) >>> Gff3().sniff( fname ) False @@ -918,7 +918,7 @@ class Gff3( Gff ): return False # Header comments may have been stripped, so inspect the data if hdr and hdr[0] and not hdr[0].startswith( '#' ): - if len(hdr) != 9: + if len(hdr) != 9: return False try: int( hdr[3] ) @@ -948,25 +948,25 @@ class Gtf( Gff ): file_ext = "gtf" column_names = [ 'Seqname', 'Source', 'Feature', 'Start', 'End', 'Score', 'Strand', 'Frame', 'Attributes' ] track_type = Interval.track_type - + """Add metadata elements""" MetadataElement( name="columns", default=9, desc="Number of columns", readonly=True, visible=False ) MetadataElement( name="column_types", default=['str','str','str','int','int','float','str','int','list'], param=metadata.ColumnTypesParameter, desc="Column types", readonly=True, visible=False ) - + def sniff( self, filename ): """ Determines whether the file is in gtf format - - GTF lines have nine required fields that must be tab-separated. The first eight GTF fields are the same as GFF. - The group field has been expanded into a list of attributes. Each attribute consists of a type/value pair. + + GTF lines have nine required fields that must be tab-separated. The first eight GTF fields are the same as GFF. + The group field has been expanded into a list of attributes. Each attribute consists of a type/value pair. Attributes must end in a semi-colon, and be separated from any following attribute by exactly one space. The attribute list must begin with the two mandatory attributes: gene_id value - A globally unique identifier for the genomic source of the sequence. transcript_id value - A globally unique identifier for the predicted transcript. - + For complete details see http://genome.ucsc.edu/FAQ/FAQformat#format4 - + >>> fname = get_test_fname( '1.bed' ) >>> Gtf().sniff( fname ) False @@ -1021,7 +1021,7 @@ class Wiggle( Tabular, _RemoteCallMixin ): data_sources = { "data": "bigwig", "index": "bigwig" } MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True, visible=False ) - + def __init__( self, **kwd ): Tabular.__init__( self, **kwd ) self.add_display_app( 'ucsc', 'display at UCSC', 'as_ucsc_display_file', 'ucsc_links' ) @@ -1032,8 +1032,8 @@ class Wiggle( Tabular, _RemoteCallMixin ): max_line_count = max( viewport_feature_count, 500 ) # maximum number of lines to check; includes comment lines if self.displayable( dataset ): try: - chrom = None - start = sys.maxint + chrom = None + start = sys.maxint end = 0 span = 1 step = None @@ -1132,7 +1132,7 @@ class Wiggle( Tabular, _RemoteCallMixin ): break if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: #we'll arbitrarily only use the first 100 data lines in this wig file to calculate tabular attributes (column types) - #this should be sufficient, except when we have mixed wig track types (bed, variable, fixed), + #this should be sufficient, except when we have mixed wig track types (bed, variable, fixed), # but those cases are not a single table that would have consistant column definitions #optional metadata values set in Tabular class will be 'None' max_data_lines = 100 @@ -1140,18 +1140,18 @@ class Wiggle( Tabular, _RemoteCallMixin ): def sniff( self, filename ): """ Determines wether the file is in wiggle format - + The .wig format is line-oriented. Wiggle data is preceeded by a track definition line, which adds a number of options for controlling the default display of this track. Following the track definition line is the track data, which can be entered in several different formats. - + The track definition line begins with the word 'track' followed by the track type. The track type with version is REQUIRED, and it currently must be wiggle_0. For example, track type=wiggle_0... - + For complete details see http://genome.ucsc.edu/goldenPath/help/wiggle.html - + >>> fname = get_test_fname( 'interval1.bed' ) >>> Wiggle().sniff( fname ) False @@ -1189,7 +1189,7 @@ class Wiggle( Tabular, _RemoteCallMixin ): t_end = math.ceil( end / resolution ) x = numpy.arange( t_start, t_end ) * resolution y = data[ t_start : t_end ] - + return zip(x.tolist(), y.tolist()) def get_track_resolution( self, dataset, start, end): range = end - start @@ -1229,7 +1229,7 @@ class CustomTrack ( Tabular ): def get_estimated_display_viewport( self, dataset, chrom_col = None, start_col = None, end_col = None ): """Return a chrom, start, stop tuple for viewing a file.""" #FIXME: only BED and WIG custom tracks are currently supported - #As per previously existing behavior, viewport will only be over the first intervals + #As per previously existing behavior, viewport will only be over the first intervals max_line_count = 100 # maximum number of lines to check; includes comment lines variable_step_wig = False chrom = None @@ -1255,7 +1255,7 @@ class CustomTrack ( Tabular ): start = int( line.rstrip( '\n\r' ).split("start=")[1].split()[0] ) return ( chrom, str( start ), str( start + span ) ) else: - variable_step_wig = True + variable_step_wig = True else: fields = line.rstrip().split( '\t' ) if len( fields ) >= 3: @@ -1296,12 +1296,12 @@ class CustomTrack ( Tabular ): def sniff( self, filename ): """ Determines whether the file is in customtrack format. - + CustomTrack files are built within Galaxy and are basically bed or interval files with the first line looking something like this. - + track name="User Track" description="User Supplied Track (from Galaxy)" color=0,0,0 visibility=1 - + >>> fname = get_test_fname( 'complete.bed' ) >>> CustomTrack().sniff( fname ) False @@ -1325,61 +1325,61 @@ class CustomTrack ( Tabular ): if not color_found or not visibility_found: return False else: return False except: return False - else: + else: try: if hdr[0] and not hdr[0].startswith( '#' ): - if len( hdr ) < 3: + if len( hdr ) < 3: return False try: int( hdr[1] ) int( hdr[2] ) - except: + except: return False - except: + except: return False return True - + class ENCODEPeak( Interval ): ''' - Human ENCODE peak format. There are both broad and narrow peak formats. + Human ENCODE peak format. There are both broad and narrow peak formats. Formats are very similar; narrow peak has an additional column, though. - + Broad peak ( http://genome.ucsc.edu/FAQ/FAQformat#format13 ): - This format is used to provide called regions of signal enrichment based + This format is used to provide called regions of signal enrichment based on pooled, normalized (interpreted) data. It is a BED 6+3 format. - + Narrow peak http://genome.ucsc.edu/FAQ/FAQformat#format12 and : This format is used to provide called peaks of signal enrichment based on pooled, normalized (interpreted) data. It is a BED6+4 format. ''' - + file_ext = "encodepeak" column_names = [ 'Chrom', 'Start', 'End', 'Name', 'Score', 'Strand', 'SignalValue', 'pValue', 'qValue', 'Peak' ] data_sources = { "data": "tabix", "index": "bigwig" } - + """Add metadata elements""" MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter ) MetadataElement( name="startCol", default=2, desc="Start column", param=metadata.ColumnParameter ) MetadataElement( name="endCol", default=3, desc="End column", param=metadata.ColumnParameter ) MetadataElement( name="strandCol", desc="Strand column (click box & select)", param=metadata.ColumnParameter, optional=True, no_value=0 ) MetadataElement( name="columns", default=3, desc="Number of columns", readonly=True, visible=False ) - + def sniff( self, filename ): return False - + class ChromatinInteractions( Interval ): ''' Chromatin interactions obtained from 3C/5C/Hi-C experiments. ''' - + file_ext = "chrint" track_type = "DiagonalHeatmapTrack" data_sources = { "data": "tabix", "index": "bigwig" } - + column_names = [ 'Chrom1', 'Start1', 'End1', 'Chrom2', 'Start2', 'End2', 'Value' ] - + """Add metadata elements""" MetadataElement( name="chrom1Col", default=1, desc="Chrom1 column", param=metadata.ColumnParameter ) MetadataElement( name="start1Col", default=2, desc="Start1 column", param=metadata.ColumnParameter ) @@ -1390,7 +1390,7 @@ class ChromatinInteractions( Interval ): MetadataElement( name="valueCol", default=7, desc="Value column", param=metadata.ColumnParameter ) MetadataElement( name="columns", default=7, desc="Number of columns", readonly=True, visible=False ) - + def sniff( self, filename ): return False diff --git a/lib/galaxy/datatypes/metadata.py b/lib/galaxy/datatypes/metadata.py index 363a9ada643..79d1084ac62 100644 --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -162,10 +162,10 @@ class MetadataSpecCollection( odict ): class MetadataParameter( object ): def __init__( self, spec ): self.spec = spec - - def get_html_field( self, value=None, context={}, other_values={}, **kwd ): + + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=value ) - + def get_html( self, value, context={}, other_values={}, **kwd ): """ The "context" is simply the metadata collection/bunch holding @@ -184,13 +184,13 @@ class MetadataParameter( object ): return checkbox.get_html() + self.get_html_field( value=value, context=context, other_values=other_values, **kwd ).get_html() else: return self.get_html_field( value=value, context=context, other_values=other_values, **kwd ).get_html() - + def to_string( self, value ): return str( value ) - + def make_copy( self, value, target_context = None, source_context = None ): return copy.deepcopy( value ) - + @classmethod def marshal ( cls, value ): """ @@ -212,7 +212,7 @@ class MetadataParameter( object ): value = self.marshal( form_value ) self.validate( value ) return value - + def wrap( self, value, session ): """ Turns a value into its usable form. @@ -288,15 +288,15 @@ class SelectParameter( MetadataParameter ): MetadataParameter.__init__( self, spec ) self.values = self.spec.get( "values" ) self.multiple = string_as_bool( self.spec.get( "multiple" ) ) - + def to_string( self, value ): if value in [ None, [] ]: return str( self.spec.no_value ) if not isinstance( value, list ): value = [value] return ",".join( map( str, value ) ) - - def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd ): + + def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd ): field = form_builder.SelectField( self.spec.name, multiple=self.multiple, display=self.spec.get("display") ) if self.values: value_list = self.values @@ -345,7 +345,7 @@ class DBKeyParameter( SelectParameter ): except KeyError: pass return super(DBKeyParameter, self).get_html_field( value, context, other_values, values, **kwd) - def get_html( self, value=None, context={}, other_values={}, values=None, **kwd): + def get_html( self, value=None, context={}, other_values={}, values=None, **kwd): try: values = kwd['trans'].db_builds except KeyError: @@ -364,54 +364,54 @@ class RangeParameter( SelectParameter ): if values is None: values = zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step )) return SelectParameter.get_html_field( self, value=value, context=context, other_values=other_values, values=values, **kwd ) - + def get_html( self, value, context={}, other_values={}, values=None, **kwd ): if values is None: values = zip( range( self.min, self.max, self.step ), range( self.min, self.max, self.step )) return SelectParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd ) - + @classmethod def marshal( cls, value ): value = SelectParameter.marshal( value ) values = [ int(x) for x in value ] return values - + class ColumnParameter( RangeParameter ): - + def get_html_field( self, value=None, context={}, other_values={}, values=None, **kwd ): if values is None and context: column_range = range( 1, ( context.columns or 0 ) + 1, 1 ) values = zip( column_range, column_range ) return RangeParameter.get_html_field( self, value=value, context=context, other_values=other_values, values=values, **kwd ) - + def get_html( self, value, context={}, other_values={}, values=None, **kwd ): if values is None and context: column_range = range( 1, ( context.columns or 0 ) + 1, 1 ) values = zip( column_range, column_range ) - return RangeParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd ) - + return RangeParameter.get_html( self, value, context=context, other_values=other_values, values=values, **kwd ) + class ColumnTypesParameter( MetadataParameter ): - + def to_string( self, value ): return ",".join( map( str, value ) ) - + class ListParameter( MetadataParameter ): def to_string( self, value ): return ",".join( [str(x) for x in value] ) class DictParameter( MetadataParameter ): - + def to_string( self, value ): return simplejson.dumps( value ) class PythonObjectParameter( MetadataParameter ): - + def to_string( self, value ): if not value: return self.spec._to_string( self.spec.no_value ) return self.spec._to_string( value ) - + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=self._to_string( value ) ) @@ -423,12 +423,12 @@ class PythonObjectParameter( MetadataParameter ): return value class FileParameter( MetadataParameter ): - + def to_string( self, value ): if not value: return str( self.spec.no_value ) return value.file_name - + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): return form_builder.TextField( self.spec.name, value=str( value.id ) ) @@ -442,7 +442,7 @@ class FileParameter( MetadataParameter ): return value mf = session.query( galaxy.model.MetadataFile ).get( value ) return mf - + def make_copy( self, value, target_context, source_context ): value = self.wrap( value, object_session( target_context.parent ) ) if value: @@ -452,13 +452,13 @@ class FileParameter( MetadataParameter ): shutil.copy( value.file_name, new_value.file_name ) return self.unwrap( new_value ) return None - + @classmethod def marshal( cls, value ): if isinstance( value, galaxy.model.MetadataFile ): value = value.id return value - + def from_external_value( self, value, parent ): """ Turns a value read from a external dict into its value to be pushed directly into the metadata dict. @@ -474,7 +474,7 @@ class FileParameter( MetadataParameter ): os.unlink( value.file_name ) value = mf.id return value - + def to_external_value( self, value ): """ Turns a value read from a metadata into its value to be pushed directly into the external dict. @@ -484,7 +484,7 @@ class FileParameter( MetadataParameter ): elif isinstance( value, MetadataTempFile ): value = MetadataTempFile.to_JSON( value ) return value - + def new_file( self, dataset = None, **kwds ): if object_session( dataset ): mf = galaxy.model.MetadataFile( name = self.spec.name, dataset = dataset, **kwds ) @@ -496,7 +496,7 @@ class FileParameter( MetadataParameter ): #we will be copying its contents into the MetadataFile objects filename after restoring from JSON #we do not include 'dataset' in the kwds passed, as from_JSON_value() will handle this for us return MetadataTempFile( **kwds ) - + #This class is used when a database file connection is not available class MetadataTempFile( object ): tmp_dir = 'database/tmp' #this should be overwritten as necessary in calling scripts @@ -550,10 +550,10 @@ class JobExternalOutputMetadataWrapper( object ): .first() #there should only be one or None return None def get_dataset_metadata_key( self, dataset ): - # Set meta can be called on library items and history items, + # Set meta can be called on library items and history items, # need to make different keys for them, since ids can overlap return "%s_%d" % ( dataset.__class__.__name__, dataset.id ) - def setup_external_metadata( self, datasets, sa_session, exec_dir=None, tmp_dir=None, dataset_files_path=None, + def setup_external_metadata( self, datasets, sa_session, exec_dir=None, tmp_dir=None, dataset_files_path=None, output_fnames=None, config_root=None, config_file=None, datatypes_config=None, job_metadata=None, kwds={} ): #fill in metadata_files_dict and return the command with args required to set metadata def __metadata_files_list_to_cmd_line( metadata_files ): @@ -582,8 +582,8 @@ class JobExternalOutputMetadataWrapper( object ): key = self.get_dataset_metadata_key( dataset ) #future note: #wonkiness in job execution causes build command line to be called more than once - #when setting metadata externally, via 'auto-detect' button in edit attributes, etc., - #we don't want to overwrite (losing the ability to cleanup) our existing dataset keys and files, + #when setting metadata externally, via 'auto-detect' button in edit attributes, etc., + #we don't want to overwrite (losing the ability to cleanup) our existing dataset keys and files, #so we will only populate the dictionary once metadata_files = self.get_output_filenames_by_dataset( dataset, sa_session ) if not metadata_files: @@ -592,17 +592,17 @@ class JobExternalOutputMetadataWrapper( object ): #we are using tempfile to create unique filenames, tempfile always returns an absolute path #we will use pathnames relative to the galaxy root, to accommodate instances where the galaxy root #is located differently, i.e. on a cluster node with a different filesystem structure - + #file to store existing dataset metadata_files.filename_in = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_in_%s_" % key ).name ) - + #FIXME: HACK #sqlalchemy introduced 'expire_on_commit' flag for sessionmaker at version 0.5x #This may be causing the dataset attribute of the dataset_association object to no-longer be loaded into memory when needed for pickling. #For now, we'll simply 'touch' dataset_association.dataset to force it back into memory. dataset.dataset #force dataset_association.dataset to be loaded before pickling #A better fix could be setting 'expire_on_commit=False' on the session, or modifying where commits occur, or ? - + cPickle.dump( dataset, open( metadata_files.filename_in, 'wb+' ) ) #file to store metadata results of set_meta() metadata_files.filename_out = abspath( tempfile.NamedTemporaryFile( dir = tmp_dir, prefix = "metadata_out_%s_" % key ).name ) @@ -630,7 +630,7 @@ class JobExternalOutputMetadataWrapper( object ): metadata_files_list.append( metadata_files ) #return command required to build return "%s %s %s %s %s %s %s %s" % ( os.path.join( exec_dir, 'set_metadata.sh' ), dataset_files_path, tmp_dir, config_root, config_file, datatypes_config, job_metadata, " ".join( map( __metadata_files_list_to_cmd_line, metadata_files_list ) ) ) - + def external_metadata_set_successfully( self, dataset, sa_session ): metadata_files = self.get_output_filenames_by_dataset( dataset, sa_session ) if not metadata_files: @@ -639,7 +639,7 @@ class JobExternalOutputMetadataWrapper( object ): if not rval: log.debug( 'setting metadata externally failed for %s %s: %s' % ( dataset.__class__.__name__, dataset.id, rstring ) ) return rval - + def cleanup_external_metadata( self, sa_session ): log.debug( 'Cleaning up external metadata files' ) for metadata_files in sa_session.query( galaxy.model.Job ).get( self.job_id ).external_output_metadata: diff --git a/lib/galaxy/datatypes/ngsindex.py b/lib/galaxy/datatypes/ngsindex.py index 9db5b28f096..ea1aa751c46 100644 --- a/lib/galaxy/datatypes/ngsindex.py +++ b/lib/galaxy/datatypes/ngsindex.py @@ -16,22 +16,22 @@ class BowtieIndex( Html ): """ MetadataElement( name="base_name", desc="base name for this index set", default='galaxy_generated_bowtie_index', set_in_upload=True, readonly=True ) MetadataElement( name="sequence_space", desc="sequence_space for this index set", default='unknown', set_in_upload=True, readonly=True ) - + file_ext = 'bowtie_index' is_binary = True composite_type = 'auto_primary_file' allow_datatype_change = False def generate_primary_file( self, dataset = None ): - """ + """ This is called only at upload to write the html file cannot rename the datasets here - they come with the default unfortunately """ return 'AutoGenerated Primary File for Composite Dataset' - + def regenerate_primary_file(self,dataset): """ - cannot do this until we are setting metadata + cannot do this until we are setting metadata """ bn = dataset.metadata.base_name flist = os.listdir(dataset.extra_files_path) @@ -65,13 +65,13 @@ class BowtieColorIndex( BowtieIndex ): Bowtie color space index """ MetadataElement( name="sequence_space", desc="sequence_space for this index set", default='color', set_in_upload=True, readonly=True ) - + file_ext = 'bowtie_color_index' - + class BowtieBaseIndex( BowtieIndex ): """ Bowtie base space index """ MetadataElement( name="sequence_space", desc="sequence_space for this index set", default='base', set_in_upload=True, readonly=True ) - + file_ext = 'bowtie_base_index' diff --git a/lib/galaxy/datatypes/qualityscore.py b/lib/galaxy/datatypes/qualityscore.py index 51bc1081371..ab67ea2d888 100644 --- a/lib/galaxy/datatypes/qualityscore.py +++ b/lib/galaxy/datatypes/qualityscore.py @@ -63,7 +63,7 @@ class QualityScoreSOLiD ( QualityScore ): except: pass return False - + def set_meta( self, dataset, **kwd ): if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: dataset.metadata.data_lines = None diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 25e8bc989f3..68151f4f2a9 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -273,11 +273,11 @@ class Registry( object ): self.to_xml_file() # Default values. if not self.datatypes_by_extension: - self.datatypes_by_extension = { + self.datatypes_by_extension = { 'ab1' : binary.Ab1(), 'axt' : sequence.Axt(), 'bam' : binary.Bam(), - 'bed' : interval.Bed(), + 'bed' : interval.Bed(), 'coverage' : coverage.LastzCoverage(), 'customtrack' : interval.CustomTrack(), 'csfasta' : sequence.csFasta(), @@ -289,7 +289,7 @@ class Registry( object ): 'gff' : interval.Gff(), 'gff3' : interval.Gff3(), 'genetrack' : tracks.GeneTrack(), - 'interval' : interval.Interval(), + 'interval' : interval.Interval(), 'laj' : images.Laj(), 'lav' : sequence.Lav(), 'maf' : sequence.Maf(), @@ -297,7 +297,7 @@ class Registry( object ): 'qualsolid' : qualityscore.QualityScoreSOLiD(), 'qualsolexa' : qualityscore.QualityScoreSolexa(), 'qual454' : qualityscore.QualityScore454(), - 'sam' : tabular.Sam(), + 'sam' : tabular.Sam(), 'scf' : binary.Scf(), 'sff' : binary.Sff(), 'tabular' : tabular.Tabular(), @@ -306,11 +306,11 @@ class Registry( object ): 'wig' : interval.Wiggle(), 'xml' : xml.GenericXml(), } - self.mimetypes_by_extension = { + self.mimetypes_by_extension = { 'ab1' : 'application/octet-stream', 'axt' : 'text/plain', 'bam' : 'application/octet-stream', - 'bed' : 'text/plain', + 'bed' : 'text/plain', 'customtrack' : 'text/plain', 'csfasta' : 'text/plain', 'eland' : 'application/octet-stream', @@ -320,7 +320,7 @@ class Registry( object ): 'gtf' : 'text/plain', 'gff' : 'text/plain', 'gff3' : 'text/plain', - 'interval' : 'text/plain', + 'interval' : 'text/plain', 'laj' : 'text/plain', 'lav' : 'text/plain', 'maf' : 'text/plain', @@ -359,7 +359,7 @@ class Registry( object ): interval.Wiggle(), images.Html(), sequence.Axt(), - interval.Bed(), + interval.Bed(), interval.CustomTrack(), interval.Gtf(), interval.Gff(), @@ -381,7 +381,7 @@ class Registry( object ): if not included: self.sniff_order.append(datatype) append_to_sniff_order() - + def get_datatype_class_by_name( self, name ): """ Return the datatype class where the datatype's `type` attribute @@ -450,7 +450,7 @@ class Registry( object ): If deactivate is False, add datatype converters from self.converters or self.proprietary_converters to the calling app's toolbox. If deactivate is True, eliminates relevant converters from the calling app's toolbox. - """ + """ if installed_repository_dict: # Load converters defined by datatypes_conf.xml included in installed tool shed repository. converters = self.proprietary_converters @@ -574,7 +574,7 @@ class Registry( object ): """Adds a tool which is used to set external metadata""" # We need to be able to add a job to the queue to set metadata. The queue will currently only accept jobs with an associated # tool. We'll create a special tool to be used for Auto-Detecting metadata; this is less than ideal, but effective - # Properly building a tool without relying on parsing an XML file is near impossible...so we'll create a temporary file + # Properly building a tool without relying on parsing an XML file is near impossible...so we'll create a temporary file tool_xml_text = """ diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 4ee5c711e5b..f63593a7915 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -31,7 +31,7 @@ log = logging.getLogger(__name__) class SequenceSplitLocations( data.Text ): """ Class storing information about a sequence file composed of multiple gzip files concatenated as - one OR an uncompressed file. In the GZIP case, each sub-file's location is stored in start and end. + one OR an uncompressed file. In the GZIP case, each sub-file's location is stored in start and end. The format of the file is JSON:: @@ -173,7 +173,7 @@ class Sequence( data.Text ): directories.append(dir) return dir - # we know how many splits and how many sequences in each. What remains is to write out instructions for the + # we know how many splits and how many sequences in each. What remains is to write out instructions for the # splitting of all the input files. To decouple the format of those instructions from this code, the exact format of # those instructions is delegated to scripts start_sequence=0 @@ -196,7 +196,7 @@ class Sequence( data.Text ): start_sequence += sequences_per_file[part_no] return directories write_split_files = classmethod(write_split_files) - + def split( cls, input_datasets, subdir_generator_function, split_params): """Split a generic sequence file (not sensible or possible, see subclasses).""" if split_params is None: @@ -216,7 +216,7 @@ class Alignment( data.Text ): return None raise NotImplementedError("Can't split generic alignment files") - + class Fasta( Sequence ): """Class representing a FASTA sequence""" file_ext = "fasta" @@ -224,13 +224,13 @@ class Fasta( Sequence ): def sniff( self, filename ): """ Determines whether the file is in fasta format - - A sequence in FASTA format consists of a single-line description, followed by lines of sequence data. - The first character of the description line is a greater-than (">") symbol in the first column. + + A sequence in FASTA format consists of a single-line description, followed by lines of sequence data. + The first character of the description line is a greater-than (">") symbol in the first column. All lines should be shorter than 80 characters - + For complete details see http://www.ncbi.nlm.nih.gov/blast/fasta.shtml - + Rules for sniffing as True: We don't care about line length (other than empty lines). @@ -246,7 +246,7 @@ class Fasta( Sequence ): This should be done through sniff order, where csfasta (currently has a null sniff function) is detected for first (stricter definition) followed sometime after by fasta We will only check that the first purported sequence is correctly formatted. - + >>> fname = get_test_fname( 'sequence.maf' ) >>> Fasta().sniff( fname ) False @@ -254,7 +254,7 @@ class Fasta( Sequence ): >>> Fasta().sniff( fname ) True """ - + try: fh = open( filename ) while True: @@ -409,7 +409,7 @@ class csFasta( Sequence ): def sniff( self, filename ): """ - Color-space sequence: + Color-space sequence: >2_15_85_F3 T213021013012303002332212012112221222112212222 @@ -443,7 +443,7 @@ class csFasta( Sequence ): except: pass return False - + def set_meta( self, dataset, **kwd ): if self.max_optional_metadata_filesize >= 0 and dataset.get_size() > self.max_optional_metadata_filesize: dataset.metadata.data_lines = None @@ -473,7 +473,7 @@ class Fastq ( Sequence ): if line and line.startswith( '#' ) and not sequences: # We don't count comment lines for sequence data types continue - if line and line.startswith( '@' ): + if line and line.startswith( '@' ): if seq_counter >= 4: # count previous block # blocks should be 4 lines long @@ -514,7 +514,7 @@ class Fastq ( Sequence ): # Check the sequence line, make sure it contains only G/C/A/T/N if not bases_regexp.match( headers[1][0] ): return False - return True + return True return False except: return False @@ -555,7 +555,7 @@ class Fastq ( Sequence ): output_name = data['output_name'] start_sequence = long(args['start_sequence']) sequence_count = long(args['num_sequences']) - + if 'toc_file' in args: toc_file = simplejson.load(open(args['toc_file'], 'r')) commands = Sequence.get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count) @@ -587,7 +587,7 @@ class FastqCSSanger( Fastq ): class Maf( Alignment ): """Class describing a Maf alignment""" file_ext = "maf" - + #Readonly and optional, users can't unset it, but if it is not set, we are generally ok; if required use a metadata validator in the tool definition MetadataElement( name="blocks", default=0, desc="Number of blocks", readonly=True, optional=True, visible=False, no_value=0 ) MetadataElement( name="species_chromosomes", desc="Species Chromosomes", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) @@ -607,7 +607,7 @@ class Maf( Alignment ): return #this is not a MAF file dataset.metadata.species = species dataset.metadata.blocks = blocks - + #write species chromosomes to a file chrom_file = dataset.metadata.species_chromosomes if not chrom_file: @@ -617,7 +617,7 @@ class Maf( Alignment ): chrom_out.write( "%s\t%s\n" % ( spec, "\t".join( chroms ) ) ) chrom_out.close() dataset.metadata.species_chromosomes = chrom_file - + index_file = dataset.metadata.maf_index if not index_file: index_file = dataset.metadata.spec['maf_index'].param.new_file( dataset = dataset ) @@ -664,18 +664,18 @@ class Maf( Alignment ): def sniff( self, filename ): """ Determines wether the file is in maf format - - The .maf format is line-oriented. Each multiple alignment ends with a blank line. - Each sequence in an alignment is on a single line, which can get quite long, but - there is no length limit. Words in a line are delimited by any white space. - Lines starting with # are considered to be comments. Lines starting with ## can + + The .maf format is line-oriented. Each multiple alignment ends with a blank line. + Each sequence in an alignment is on a single line, which can get quite long, but + there is no length limit. Words in a line are delimited by any white space. + Lines starting with # are considered to be comments. Lines starting with ## can be ignored by most programs, but contain meta-data of one form or another. - - The first line of a .maf file begins with ##maf. This word is followed by white-space-separated + + The first line of a .maf file begins with ##maf. This word is followed by white-space-separated variable=value pairs. There should be no white space surrounding the "=". - + For complete details see http://genome.ucsc.edu/FAQ/FAQformat#format5 - + >>> fname = get_test_fname( 'sequence.maf' ) >>> Maf().sniff( fname ) True @@ -695,11 +695,11 @@ class Maf( Alignment ): class MafCustomTrack( data.Text ): file_ext = "mafcustomtrack" - + MetadataElement( name="vp_chromosome", default='chr1', desc="Viewport Chromosome", readonly=True, optional=True, visible=False, no_value='' ) MetadataElement( name="vp_start", default='1', desc="Viewport Start", readonly=True, optional=True, visible=False, no_value='' ) MetadataElement( name="vp_end", default='100', desc="Viewport End", readonly=True, optional=True, visible=False, no_value='' ) - + def set_meta( self, dataset, overwrite = True, **kwd ): """ Parses and sets viewport metadata from MAF file. @@ -722,7 +722,7 @@ class MafCustomTrack( data.Text ): forward_strand_end = max( forward_strand_end, ref_comp.forward_strand_end ) if i > max_block_check: break - + if forward_strand_end > forward_strand_start: dataset.metadata.vp_chromosome = chrom dataset.metadata.vp_start = forward_strand_start @@ -733,7 +733,7 @@ class MafCustomTrack( data.Text ): class Axt( data.Text ): """Class describing an axt alignment""" - + # gvk- 11/19/09 - This is really an alignment, but we no longer have tools that use this data type, and it is # here simply for backward compatibility ( although it is still in the datatypes registry ). Subclassing # from data.Text eliminates managing metadata elements inherited from the Alignemnt class. @@ -743,21 +743,21 @@ class Axt( data.Text ): def sniff( self, filename ): """ Determines whether the file is in axt format - - axt alignment files are produced from Blastz, an alignment tool available from Webb Miller's lab + + axt alignment files are produced from Blastz, an alignment tool available from Webb Miller's lab at Penn State University. - + Each alignment block in an axt file contains three lines: a summary line and 2 sequence lines. Blocks are separated from one another by blank lines. - + The summary line contains chromosomal position and size information about the alignment. It consists of 9 required fields. - + The sequence lines contain the sequence of the primary assembly (line 2) and aligning assembly (line 3) with inserts. Repeats are indicated by lower-case letters. - + For complete details see http://genome.ucsc.edu/goldenPath/help/axt.html - + >>> fname = get_test_fname( 'alignment.axt' ) >>> Axt().sniff( fname ) True @@ -796,12 +796,12 @@ class Lav( data.Text ): def sniff( self, filename ): """ Determines whether the file is in lav format - + LAV is an alignment format developed by Webb Miller's group. It is the primary output format for BLASTZ. The first line of a .lav file begins with #:lav. - + For complete details see http://www.bioperl.org/wiki/LAV_alignment_format - + >>> fname = get_test_fname( 'alignment.lav' ) >>> Lav().sniff( fname ) True diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index cee2d02f90b..7d64e386c1f 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -142,7 +142,7 @@ def sep2tabs( fname, in_place=True, patt="\\s+" ): if i is None: i = 0 else: - i += 1 + i += 1 if in_place: shutil.move( temp_name, fname ) # Return number of lines in file. @@ -327,7 +327,7 @@ def guess_ext( fname, sniff_order=None, is_multi_byte=False ): for hdr in headers: for char in hdr: #old behavior had 'char' possibly having length > 1, - #need to determine when/if this occurs + #need to determine when/if this occurs is_binary = util.is_binary( char ) if is_binary: break diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 5b94d8a89ba..11c13c73775 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -621,7 +621,7 @@ class Pileup( Tabular ): return True except: return False - + # ------------- Dataproviders @dataproviders.decorators.dataprovider_factory( 'genomic-region', dataproviders.dataset.GenomicRegionDataProvider.settings ) diff --git a/lib/galaxy/datatypes/tracks.py b/lib/galaxy/datatypes/tracks.py index cc84b287c81..5b99b21e031 100644 --- a/lib/galaxy/datatypes/tracks.py +++ b/lib/galaxy/datatypes/tracks.py @@ -15,7 +15,7 @@ log = logging.getLogger(__name__) class GeneTrack( binary.Binary ): file_ext = "genetrack" - + def __init__(self, **kwargs): super( GeneTrack, self ).__init__( **kwargs ) # self.add_display_app( 'genetrack', 'View in', '', 'genetrack_link' ) diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py index 98f4a571f2d..cc8f20a1e9a 100644 --- a/lib/galaxy/datatypes/util/gff_util.py +++ b/lib/galaxy/datatypes/util/gff_util.py @@ -9,7 +9,7 @@ from bx.tabular.io import Header, Comment from galaxy.util.odict import odict class GFFInterval( GenomicInterval ): - """ + """ A GFF interval, including attributes. If file is strictly a GFF file, only attribute is 'group.' """ @@ -26,7 +26,7 @@ class GFFInterval( GenomicInterval ): if unknown_strand: self.strand = '.' self.fields[ strand_col ] = '.' - + # Handle feature, score column. self.feature_col = feature_col if self.feature_col >= self.nfields: @@ -36,14 +36,14 @@ class GFFInterval( GenomicInterval ): if self.score_col >= self.nfields: raise MissingFieldError( "No field for score_col (%d)" % score_col ) self.score = self.fields[ self.score_col ] - + # GFF attributes. self.attributes = parse_gff_attributes( fields[8] ) - + def copy( self ): - return GFFInterval(self.reader, list( self.fields ), self.chrom_col, self.feature_col, self.start_col, + return GFFInterval(self.reader, list( self.fields ), self.chrom_col, self.feature_col, self.start_col, self.end_col, self.strand_col, self.score_col, self.strand) - + class GFFFeature( GFFInterval ): """ A GFF feature, which can include multiple intervals. @@ -68,14 +68,14 @@ class GFFFeature( GFFInterval ): self.start = interval.start if interval.end > self.end: self.end = interval.end - + def name( self ): """ Returns feature's name. """ name = None # Preference for name: GTF, GFF3, GFF. - for attr_name in [ - # GTF: - 'transcript_id', 'gene_id', + for attr_name in [ + # GTF: + 'transcript_id', 'gene_id', # GFF3: 'ID', 'id', # GFF (TODO): @@ -84,29 +84,29 @@ class GFFFeature( GFFInterval ): if name is not None: break return name - + def copy( self ): intervals_copy = [] for interval in self.intervals: intervals_copy.append( interval.copy() ) return GFFFeature(self.reader, self.chrom_col, self.feature_col, self.start_col, self.end_col, self.strand_col, self.score_col, self.strand, intervals=intervals_copy ) - + def lines( self ): lines = [] for interval in self.intervals: lines.append( '\t'.join( interval.fields ) ) return lines - - + + class GFFIntervalToBEDReaderWrapper( NiceReaderWrapper ): - """ - Reader wrapper that reads GFF intervals/lines and automatically converts - them to BED format. """ - + Reader wrapper that reads GFF intervals/lines and automatically converts + them to BED format. + """ + def parse_row( self, line ): - # HACK: this should return a GFF interval, but bx-python operations + # HACK: this should return a GFF interval, but bx-python operations # require GenomicInterval objects and subclasses will not work. interval = GenomicInterval( self, line.split( "\t" ), self.chrom_col, self.start_col, \ self.end_col, self.strand_col, self.default_strand, \ @@ -117,17 +117,17 @@ class GFFIntervalToBEDReaderWrapper( NiceReaderWrapper ): class GFFReaderWrapper( NiceReaderWrapper ): """ Reader wrapper for GFF files. - + Wrapper has two major functions: - 1. group entries for GFF file (via group column), GFF3 (via id attribute), + 1. group entries for GFF file (via group column), GFF3 (via id attribute), or GTF (via gene_id/transcript id); - 2. convert coordinates from GFF format--starting and ending coordinates - are 1-based, closed--to the 'traditional'/BED interval format--0 based, - half-open. This is useful when using GFF files as inputs to tools that + 2. convert coordinates from GFF format--starting and ending coordinates + are 1-based, closed--to the 'traditional'/BED interval format--0 based, + half-open. This is useful when using GFF files as inputs to tools that expect traditional interval format. """ - + def __init__( self, reader, chrom_col=0, feature_col=2, start_col=3, \ end_col=4, strand_col=6, score_col=5, fix_strand=False, convert_to_bed_coord=False, **kwargs ): NiceReaderWrapper.__init__( self, reader, chrom_col=chrom_col, start_col=start_col, end_col=end_col, \ @@ -139,20 +139,20 @@ class GFFReaderWrapper( NiceReaderWrapper ): self.cur_offset = 0 self.seed_interval = None self.seed_interval_line_len = 0 - + def parse_row( self, line ): interval = GFFInterval( self, line.split( "\t" ), self.chrom_col, self.feature_col, \ self.start_col, self.end_col, self.strand_col, self.score_col, \ self.default_strand, fix_strand=self.fix_strand ) return interval - + def next( self ): """ Returns next GFFFeature. """ - + # # Helper function. # - + def handle_parse_error( parse_error ): """ Actions to take when ParseError found. """ if self.outstream: @@ -162,18 +162,18 @@ class GFFReaderWrapper( NiceReaderWrapper ): # no reason to stuff an entire bad file into memmory if self.skipped < 10: self.skipped_lines.append( ( self.linenum, self.current_line, str( e ) ) ) - - # For debugging, uncomment this to propogate parsing exceptions up. - # I.e. the underlying reason for an unexpected StopIteration exception - # can be found by uncommenting this. + + # For debugging, uncomment this to propogate parsing exceptions up. + # I.e. the underlying reason for an unexpected StopIteration exception + # can be found by uncommenting this. # raise e - + # # Get next GFFFeature # raw_size = self.seed_interval_line_len - # If there is no seed interval, set one. Also, if there are no more + # If there is no seed interval, set one. Also, if there are no more # intervals to read, this is where iterator dies. if not self.seed_interval: while not self.seed_interval: @@ -184,7 +184,7 @@ class GFFReaderWrapper( NiceReaderWrapper ): # TODO: When no longer supporting python 2.4 use finally: #finally: raw_size += len( self.current_line ) - + # If header or comment, clear seed interval and return it with its size. if isinstance( self.seed_interval, ( Header, Comment ) ): return_val = self.seed_interval @@ -192,7 +192,7 @@ class GFFReaderWrapper( NiceReaderWrapper ): self.seed_interval = None self.seed_interval_line_len = 0 return return_val - + # Initialize feature identifier from seed. feature_group = self.seed_interval.attributes.get( 'group', None ) # For GFF # For GFF3 @@ -210,7 +210,7 @@ class GFFReaderWrapper( NiceReaderWrapper ): interval = GenomicIntervalReader.next( self ) raw_size += len( self.current_line ) except StopIteration, e: - # No more intervals to read, but last feature needs to be + # No more intervals to read, but last feature needs to be # returned. interval = None raw_size += len( self.current_line ) @@ -222,11 +222,11 @@ class GFFReaderWrapper( NiceReaderWrapper ): # TODO: When no longer supporting python 2.4 use finally: #finally: #raw_size += len( self.current_line ) - + # Ignore comments. if isinstance( interval, Comment ): continue - + # Determine if interval is part of feature. part_of = False group = interval.attributes.get( 'group', None ) @@ -242,20 +242,20 @@ class GFFReaderWrapper( NiceReaderWrapper ): transcript_id = interval.attributes.get( 'transcript_id', None ) if transcript_id and transcript_id == feature_transcript_id: part_of = True - + # If interval is not part of feature, clean up and break. if not part_of: # Adjust raw size because current line is not part of feature. raw_size -= len( self.current_line ) break - + # Interval associated with feature. feature_intervals.append( interval ) - + # Last interval read is the seed for the next interval. self.seed_interval = interval self.seed_interval_line_len = len( self.current_line ) - + # Return feature. feature = GFFFeature( self, self.chrom_col, self.feature_col, self.start_col, \ self.end_col, self.strand_col, self.score_col, \ @@ -267,12 +267,12 @@ class GFFReaderWrapper( NiceReaderWrapper ): convert_gff_coords_to_bed( feature ) return feature - + def convert_bed_coords_to_gff( interval ): """ - Converts an interval object's coordinates from BED format to GFF format. - Accepted object types include GenomicInterval and list (where the first - element in the list is the interval's start, and the second element is + Converts an interval object's coordinates from BED format to GFF format. + Accepted object types include GenomicInterval and list (where the first + element in the list is the interval's start, and the second element is the interval's end). """ if isinstance( interval, GenomicInterval ): @@ -283,12 +283,12 @@ def convert_bed_coords_to_gff( interval ): elif type ( interval ) is list: interval[ 0 ] += 1 return interval - + def convert_gff_coords_to_bed( interval ): """ - Converts an interval object's coordinates from GFF format to BED format. + Converts an interval object's coordinates from GFF format to BED format. Accepted object types include GFFFeature, GenomicInterval, and list (where - the first element in the list is the interval's start, and the second + the first element in the list is the interval's start, and the second element is the interval's end). """ if isinstance( interval, GenomicInterval ): @@ -299,22 +299,22 @@ def convert_gff_coords_to_bed( interval ): elif type ( interval ) is list: interval[ 0 ] -= 1 return interval - + def parse_gff_attributes( attr_str ): """ - Parses a GFF/GTF attribute string and returns a dictionary of name-value - pairs. The general format for a GFF3 attributes string is + Parses a GFF/GTF attribute string and returns a dictionary of name-value + pairs. The general format for a GFF3 attributes string is name1=value1;name2=value2 - The general format for a GTF attribute string is + The general format for a GTF attribute string is name1 "value1" ; name2 "value2" The general format for a GFF attribute string is a single string that - denotes the interval's group; in this case, method returns a dictionary + denotes the interval's group; in this case, method returns a dictionary with a single key-value pair, and key name is 'group' - """ + """ attributes_list = attr_str.split(";") attributes = {} for name_value_pair in attributes_list: @@ -334,16 +334,16 @@ def parse_gff_attributes( attr_str ): # Need to strip double quote from values value = pair[1].strip(" \"") attributes[ name ] = value - + if len( attributes ) == 0: - # Could not split attributes string, so entire string must be + # Could not split attributes string, so entire string must be # 'group' attribute. This is the case for strictly GFF files. attributes['group'] = attr_str return attributes - + def gff_attributes_to_str( attrs, gff_format ): """ - Convert GFF attributes to string. Supported formats are GFF3, GTF. + Convert GFF attributes to string. Supported formats are GFF3, GTF. """ if gff_format == 'GTF': format_string = '%s "%s"' @@ -363,7 +363,7 @@ def gff_attributes_to_str( attrs, gff_format ): for name, value in attrs.items(): attrs_strs.append( format_string % ( name, value ) ) return " ; ".join( attrs_strs ) - + def read_unordered_gtf( iterator, strict=False ): """ Returns GTF features found in an iterator. GTF lines need not be ordered @@ -383,7 +383,7 @@ def read_unordered_gtf( iterator, strict=False ): # datasources, such as RefGenes in UCSC. key_fn = lambda fields: fields[0] + '_' + get_transcript_id( fields ) - + # Aggregate intervals by transcript_id and collect comments. feature_intervals = odict() comments = [] @@ -399,7 +399,7 @@ def read_unordered_gtf( iterator, strict=False ): feature = [] feature_intervals[ line_key ] = feature feature.append( GFFInterval( None, line.split( '\t' ) ) ) - + # Create features. chroms_features = {} for count, intervals in enumerate( feature_intervals.values() ): @@ -409,7 +409,7 @@ def read_unordered_gtf( iterator, strict=False ): if feature.chrom not in chroms_features: chroms_features[ feature.chrom ] = [] chroms_features[ feature.chrom ].append( feature ) - + # Sort features by chrom, start position. chroms_features_sorted = [] for chrom_features in chroms_features.values(): @@ -417,10 +417,10 @@ def read_unordered_gtf( iterator, strict=False ): chroms_features_sorted.sort( lambda a,b: cmp( a[0].chrom, b[0].chrom ) ) for features in chroms_features_sorted: features.sort( lambda a,b: cmp( a.start, b.start ) ) - + # Yield comments first, then features. - # FIXME: comments can appear anywhere in file, not just the beginning. - # Ideally, then comments would be associated with features and output + # FIXME: comments can appear anywhere in file, not just the beginning. + # Ideally, then comments would be associated with features and output # just before feature/line. for comment in comments: yield comment @@ -428,4 +428,4 @@ def read_unordered_gtf( iterator, strict=False ): for chrom_features in chroms_features_sorted: for feature in chrom_features: yield feature - + diff --git a/lib/galaxy/datatypes/util/image_util.py b/lib/galaxy/datatypes/util/image_util.py index 5e2e7964d2a..81855f02b65 100644 --- a/lib/galaxy/datatypes/util/image_util.py +++ b/lib/galaxy/datatypes/util/image_util.py @@ -34,13 +34,13 @@ def image_type( filename, image=None ): return format def check_image_type( filename, types, image=None ): format = image_type( filename, image ) - # First check if we can use PIL + # First check if we can use PIL if format in types: return True return False def get_image_ext ( file_path, image ): #determine ext - format = image_type( file_path, image ) + format = image_type( file_path, image ) if format in [ 'JPG','JPEG' ]: return 'jpg' if format == 'PNG': diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index c998b38f57e..131ed96aa4b 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -25,7 +25,7 @@ class GenericXml( data.Text ): def sniff( self, filename ): """ Determines whether the file is XML or not - + >>> fname = get_test_fname( 'megablast_xml_parser_test1.blastxml' ) >>> GenericXml().sniff( fname ) True @@ -37,7 +37,7 @@ class GenericXml( data.Text ): handle = open(filename) line = handle.readline() handle.close() - + #TODO - Is there a more robust way to do this? return line.startswith('"; """ return get_form_template(cls.name, cls.verbose_name, form, "This action will rename the result dataset.") - + @classmethod def get_short_str(cls, pja): return "Delete this dataset after creation." @@ -324,12 +324,12 @@ class ColumnSetAction(DefaultJobAction): class SetMetadataAction(DefaultJobAction): name = "SetMetadataAction" # DBTODO Setting of Metadata is currently broken and disabled. It should not be used (yet). - + @classmethod def execute(cls, app, sa_session, action, job, replacement_dict): for data in job.output_datasets: data.set_metadata( action.action_arguments['newtype'] ) - + @classmethod def get_config_form(cls, trans): # dt_list = "" @@ -360,10 +360,10 @@ class SetMetadataAction(DefaultJobAction): class ActionBox(object): - + actions = { "RenameDatasetAction" : RenameDatasetAction, "HideDatasetAction" : HideDatasetAction, - "ChangeDatatypeAction": ChangeDatatypeAction, + "ChangeDatatypeAction": ChangeDatatypeAction, "ColumnSetAction" : ColumnSetAction, "EmailAction" : EmailAction, # "SetMetadataAction" : SetMetadataAction, @@ -400,7 +400,7 @@ class ActionBox(object): # Not pja stuff. pass return to_json_string(npd) - + @classmethod def get_add_list(cls): addlist = "" return addlist - + @classmethod def get_forms(cls, trans): forms = "" for action in ActionBox.actions: forms += ActionBox.actions[action].get_config_form(trans) return forms - + @classmethod def execute(cls, app, sa_session, pja, job, replacement_dict = None): if ActionBox.actions.has_key(pja.action_type): diff --git a/lib/galaxy/jobs/deferred/__init__.py b/lib/galaxy/jobs/deferred/__init__.py index 6ab341279e6..23380fac36c 100644 --- a/lib/galaxy/jobs/deferred/__init__.py +++ b/lib/galaxy/jobs/deferred/__init__.py @@ -11,7 +11,7 @@ log = logging.getLogger( __name__ ) class DeferredJobQueue( object ): job_states = Bunch( READY = 'ready', - WAIT = 'wait', + WAIT = 'wait', INVALID = 'invalid' ) def __init__( self, app ): self.app = app diff --git a/lib/galaxy/jobs/deferred/data_transfer.py b/lib/galaxy/jobs/deferred/data_transfer.py index 762a19f8908..f1b4969e707 100644 --- a/lib/galaxy/jobs/deferred/data_transfer.py +++ b/lib/galaxy/jobs/deferred/data_transfer.py @@ -80,7 +80,7 @@ class DataTransfer( object ): job.state = self.app.model.DeferredJob.states.OK self.sa_session.add( job ) self.sa_session.flush() - # TODO: Error handling: failure executing, or errors returned from the manager + # TODO: Error handling: failure executing, or errors returned from the manager if job.params[ 'type' ] == 'finish_transfer': protocol = job.params[ 'protocol' ] # Update the state of the relevant SampleDataset @@ -92,7 +92,7 @@ class DataTransfer( object ): elif protocol in [ 'scp' ]: # In this case, job.params will be a dictionary that contains a key named 'result'. The value # of the result key is a dictionary that looks something like: - # {'sample_dataset_id': '8', 'status': 'Not started', 'protocol': 'scp', 'name': '3.bed', + # {'sample_dataset_id': '8', 'status': 'Not started', 'protocol': 'scp', 'name': '3.bed', # 'file_path': '/data/library/3.bed', 'host': '127.0.0.1', 'sample_id': 8, 'external_service_id': 2, # 'local_path': '/tmp/kjl2Ss4', 'password': 'galaxy', 'user_name': 'gvk', 'error_msg': '', 'size': '8.0K'} try: @@ -213,7 +213,7 @@ class DataTransfer( object ): # result_dict looks something like: # {'url': '127.0.0.1/data/filtered_subreads.fa', 'name': 'Filtered reads'} # Check if the new status is a valid transfer status - valid_statuses = [ v[1] for v in self.app.model.SampleDataset.transfer_status.items() ] + valid_statuses = [ v[1] for v in self.app.model.SampleDataset.transfer_status.items() ] # TODO: error checking on valid new_status value if protocol in [ 'http', 'https' ]: sample_dataset = self.sa_session.query( self.app.model.SampleDataset ) \ diff --git a/lib/galaxy/jobs/deferred/genome_index.py b/lib/galaxy/jobs/deferred/genome_index.py index 7572df5b109..e519a9990d6 100644 --- a/lib/galaxy/jobs/deferred/genome_index.py +++ b/lib/galaxy/jobs/deferred/genome_index.py @@ -14,7 +14,7 @@ log = logging.getLogger( __name__ ) __all__ = [ 'GenomeIndexPlugin' ] class GenomeIndexPlugin( DataTransfer ): - + def __init__( self, app ): super( GenomeIndexPlugin, self ).__init__( app ) self.app = app @@ -28,11 +28,11 @@ class GenomeIndexPlugin( DataTransfer ): self.sa_session.flush() log.debug( 'Job created, id %d' % deferred.id ) return deferred.id - + def check_job( self, job ): log.debug( 'Job check' ) return 'ready' - + def run_job( self, job ): incoming = dict( path=os.path.abspath( job.params[ 'path' ] ), indexer=job.params[ 'indexes' ][0], user=job.params[ 'user' ] ) indexjob = self.tool.execute( self, set_output_hid=False, history=None, incoming=incoming, transfer=None, deferred=job ) diff --git a/lib/galaxy/jobs/deferred/genome_transfer.py b/lib/galaxy/jobs/deferred/genome_transfer.py index 0393710586a..c88f499f7d5 100644 --- a/lib/galaxy/jobs/deferred/genome_transfer.py +++ b/lib/galaxy/jobs/deferred/genome_transfer.py @@ -25,9 +25,9 @@ log = logging.getLogger( __name__ ) __all__ = [ 'GenomeTransferPlugin' ] class GenomeTransferPlugin( DataTransfer ): - + locations = {} - + def __init__( self, app ): super( GenomeTransferPlugin, self ).__init__( app ) self.app = app @@ -39,7 +39,7 @@ class GenomeTransferPlugin( DataTransfer ): table = node.get('name') location = node.findall('file')[0].get('path') self.locations[table] = location - + def create_job( self, trans, url, dbkey, intname, indexes ): job = trans.app.transfer_manager.new( protocol='http', url=url ) params = dict( user=trans.user.id, transfer_job_id=job.id, protocol='http', type='init_transfer', url=url, dbkey=dbkey, indexes=indexes, intname=intname, liftover=None ) @@ -47,9 +47,9 @@ class GenomeTransferPlugin( DataTransfer ): self.sa_session.add( deferred ) self.sa_session.flush() return deferred.id - + def check_job( self, job ): - if job.params['type'] == 'init_transfer': + if job.params['type'] == 'init_transfer': if not hasattr(job, 'transfer_job'): job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) else: @@ -73,9 +73,9 @@ class GenomeTransferPlugin( DataTransfer ): else: log.error( "An error occurred while downloading from %s" % job.params[ 'url' ] ) return self.job_states.INVALID - elif job.params[ 'type' ] == 'extract_transfer': + elif job.params[ 'type' ] == 'extract_transfer': return self.job_states.READY - + def get_job_status( self, jobid ): job = self.sa_session.query( self.app.model.DeferredJob ).get( int( jobid ) ) if 'transfer_job_id' in job.params: @@ -84,7 +84,7 @@ class GenomeTransferPlugin( DataTransfer ): else: self.sa_session.refresh( job.transfer_job ) return job - + def run_job( self, job ): params = job.params dbkey = params[ 'dbkey' ] @@ -214,7 +214,7 @@ class GenomeTransferPlugin( DataTransfer ): self.sa_session.add( job ) self.sa_session.flush() return self.app.model.DeferredJob.states.OK - + def _check_compress( self, filepath ): retval = '' if tarfile.is_tarfile( filepath ): @@ -228,7 +228,7 @@ class GenomeTransferPlugin( DataTransfer ): if is_gzipped and is_valid: return retval + 'gzip' return None - + def _add_line( self, locfile, newline ): filepath = self.locations[ locfile ] origlines = [] @@ -247,4 +247,4 @@ class GenomeTransferPlugin( DataTransfer ): output.extend( origlines ) with open( filepath, 'w+' ) as destfile: destfile.write( '\n'.join( output ) ) - + diff --git a/lib/galaxy/jobs/deferred/liftover_transfer.py b/lib/galaxy/jobs/deferred/liftover_transfer.py index fe921f5ffb9..7765c2268e6 100644 --- a/lib/galaxy/jobs/deferred/liftover_transfer.py +++ b/lib/galaxy/jobs/deferred/liftover_transfer.py @@ -24,26 +24,26 @@ log = logging.getLogger( __name__ ) __all__ = [ 'LiftOverTransferPlugin' ] class LiftOverTransferPlugin( DataTransfer ): - + locations = {} - + def __init__( self, app ): super( LiftOverTransferPlugin, self ).__init__( app ) self.app = app self.sa_session = app.model.context.current - + def create_job( self, trans, url, dbkey, from_genome, to_genome, destfile, parentjob ): job = trans.app.transfer_manager.new( protocol='http', url=url ) - params = dict( user=trans.user.id, transfer_job_id=job.id, protocol='http', - type='init_transfer', dbkey=dbkey, from_genome=from_genome, + params = dict( user=trans.user.id, transfer_job_id=job.id, protocol='http', + type='init_transfer', dbkey=dbkey, from_genome=from_genome, to_genome=to_genome, destfile=destfile, parentjob=parentjob ) deferred = trans.app.model.DeferredJob( state = self.app.model.DeferredJob.states.NEW, plugin = 'LiftOverTransferPlugin', params = params ) self.sa_session.add( deferred ) self.sa_session.flush() return deferred.id - + def check_job( self, job ): - if job.params['type'] == 'init_transfer': + if job.params['type'] == 'init_transfer': if not hasattr(job, 'transfer_job'): job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) else: @@ -79,13 +79,13 @@ class LiftOverTransferPlugin( DataTransfer ): else: log.error( "An error occurred while downloading from %s" % job.transfer_job.params[ 'url' ] ) return self.job_states.INVALID - elif job.params[ 'type' ] == 'extract_transfer': + elif job.params[ 'type' ] == 'extract_transfer': return self.job_states.READY - + def get_job_status( self, jobid ): job = self.sa_session.query( self.app.model.DeferredJob ).get( int( jobid ) ) return job - + def run_job( self, job ): params = job.params dbkey = params[ 'dbkey' ] @@ -144,7 +144,7 @@ class LiftOverTransferPlugin( DataTransfer ): self.sa_session.add( transfer ) self.sa_session.flush() return self.app.model.DeferredJob.states.OK - + def _add_line( self, newline ): filepath = 'tool-data/liftOver.loc' origlines = [] @@ -155,4 +155,4 @@ class LiftOverTransferPlugin( DataTransfer ): origlines.append( newline ) with open( filepath, 'w+' ) as destfile: destfile.write( '\n'.join( origlines ) ) - + diff --git a/lib/galaxy/jobs/deferred/manual_data_transfer.py b/lib/galaxy/jobs/deferred/manual_data_transfer.py index c9bb499f4fd..4ce987c79bf 100644 --- a/lib/galaxy/jobs/deferred/manual_data_transfer.py +++ b/lib/galaxy/jobs/deferred/manual_data_transfer.py @@ -24,7 +24,7 @@ class ManualDataTransferPlugin( DataTransfer ): user_name = external_service.form_values.content[ 'user_name' ] password = external_service.form_values.content[ 'password' ] # TODO: In the future, we may want to implement a way for the user to associate a selected file with one of - # the run outputs configured in the section of the external service config file. The + # the run outputs configured in the section of the external service config file. The # following was a first pass at implementing something (the datatype was included in the sample_dataset_dict), # but without a way for the user to associate stuff it's useless. However, allowing the user this ability may # open a can of worms, so maybe we shouldn't do it??? diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 2aa90f750f7..e6cb96b4d33 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -585,14 +585,14 @@ class DefaultJobDispatcher( object ): Stop the given job. The input variable job may be either a Job or a Task. """ # The Job and Task classes have been modified so that their accessors - # will return the appropriate value. + # will return the appropriate value. # Note that Jobs and Tasks have runner_names, which are distinct from # the job_runner_name and task_runner_name. if ( isinstance( job, model.Job ) ): log.debug( "Stopping job %d:", job.get_id() ) elif( isinstance( job, model.Task ) ): - log.debug( "Stopping job %d, task %d" + log.debug( "Stopping job %d, task %d" % ( job.get_job().get_id(), job.get_id() ) ) else: log.debug( "Unknown job to stop" ) @@ -605,7 +605,7 @@ class DefaultJobDispatcher( object ): if ( isinstance( job, model.Job ) ): log.debug( "stopping job %d in %s runner" %( job.get_id(), runner_name ) ) elif ( isinstance( job, model.Task ) ): - log.debug( "Stopping job %d, task %d in %s runner" + log.debug( "Stopping job %d, task %d in %s runner" % ( job.get_job().get_id(), job.get_id(), runner_name ) ) try: self.job_runners[runner_name].stop_job( job ) diff --git a/lib/galaxy/jobs/mapper.py b/lib/galaxy/jobs/mapper.py index 885fd6b21bf..e6d5b7c4f5c 100644 --- a/lib/galaxy/jobs/mapper.py +++ b/lib/galaxy/jobs/mapper.py @@ -1,6 +1,6 @@ import logging import inspect -import os +import os log = logging.getLogger( __name__ ) @@ -53,16 +53,16 @@ class JobRunnerMapper( object ): rule_module_name = "galaxy.jobs.rules.%s" % fname[:-len(".py")] names.append( rule_module_name ) return names - + def __invoke_expand_function( self, expand_function ): function_arg_names = inspect.getargspec( expand_function ).args - possible_args = { "job_id" : self.job_wrapper.job_id, + possible_args = { "job_id" : self.job_wrapper.job_id, "tool" : self.job_wrapper.tool, "tool_id" : self.job_wrapper.tool.id, "job_wrapper" : self.job_wrapper, "app" : self.job_wrapper.app } - + actual_args = {} # Populate needed args @@ -82,7 +82,7 @@ class JobRunnerMapper( object ): if "user" in function_arg_names: actual_args[ "user" ] = user - + if "user_email" in function_arg_names: actual_args[ "user_email" ] = user_email @@ -109,7 +109,7 @@ class JobRunnerMapper( object ): for tool_id in self.job_wrapper.tool.all_ids: if self.__last_rule_module_with_function( tool_id ): expand_function_name = tool_id - break + break return expand_function_name def __get_expand_function( self, expand_function_name ): @@ -127,7 +127,7 @@ class JobRunnerMapper( object ): if hasattr( rule_module, function_name ): return rule_module return None - + def __handle_dynamic_job_destination( self, destination ): expand_type = destination.params.get('type', "python") if expand_type == "python": diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 88e0069c3a2..4a86cf27055 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -73,7 +73,7 @@ class BaseJobRunner( object ): def mark_as_queued(self, job_wrapper): self.work_queue.put( ( self.queue_job, job_wrapper ) ) - + def shutdown( self ): """Attempts to gracefully shut down the worker threads """ @@ -112,7 +112,7 @@ class BaseJobRunner( object ): job_wrapper.cleanup() return False elif job_state != model.Job.states.QUEUED: - log.info( "(%d) Job is in state %s, skipping execution" % ( job_id, job_state ) ) + log.info( "(%d) Job is in state %s, skipping execution" % ( job_id, job_state ) ) # cleanup may not be safe in all states return False @@ -162,29 +162,29 @@ class BaseJobRunner( object ): commands = "%s &> %s; " % ( job_wrapper.version_string_cmd, job_wrapper.get_version_string_path() ) + commands # prepend getting input files (if defined) if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None: - commands = "; ".join( job_wrapper.prepare_input_files_cmds + [ commands ] ) + commands = "; ".join( job_wrapper.prepare_input_files_cmds + [ commands ] ) # Prepend dependency injection if job_wrapper.dependency_shell_commands: - commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] ) + commands = "; ".join( job_wrapper.dependency_shell_commands + [ commands ] ) # Append commands to copy job outputs based on from_work_dir attribute. if include_work_dir_outputs: work_dir_outputs = self.get_work_dir_outputs( job_wrapper ) if work_dir_outputs: - commands += "; " + "; ".join( [ "if [ -f %s ] ; then cp %s %s ; fi" % + commands += "; " + "; ".join( [ "if [ -f %s ] ; then cp %s %s ; fi" % ( source_file, source_file, destination ) for ( source_file, destination ) in work_dir_outputs ] ) # Append metadata setting commands, we don't want to overwrite metadata # that was copied over in init_meta(), as per established behavior if include_metadata: commands += "; cd %s; " % os.path.abspath( os.getcwd() ) - commands += job_wrapper.setup_external_metadata( + commands += job_wrapper.setup_external_metadata( exec_dir = os.path.abspath( os.getcwd() ), tmp_dir = job_wrapper.working_directory, dataset_files_path = self.app.model.Dataset.file_path, output_fnames = job_wrapper.get_output_fnames(), set_extension = False, - kwds = { 'overwrite' : False } ) + kwds = { 'overwrite' : False } ) return commands def get_work_dir_outputs( self, job_wrapper ): @@ -205,7 +205,7 @@ class BaseJobRunner( object ): return os.path.commonprefix( [ file, directory ] ) == directory - # Set up dict of dataset id --> output path; output path can be real or + # Set up dict of dataset id --> output path; output path can be real or # false depending on outputs_to_working_directory output_paths = {} for dataset_path in job_wrapper.get_output_fnames(): @@ -348,7 +348,7 @@ class AsynchronousJobRunner( BaseJobRunner ): while 1: # Take any new watched jobs and put them on the monitor list try: - while 1: + while 1: async_job_state = self.monitor_queue.get_nowait() if async_job_state is STOP_SIGNAL: # TODO: This is where any cleanup would occur @@ -422,7 +422,7 @@ class AsynchronousJobRunner( BaseJobRunner ): which_try += 1 try: - # This should be an 8-bit exit code, but read ahead anyway: + # This should be an 8-bit exit code, but read ahead anyway: exit_code_str = file( job_state.exit_code_file, "r" ).read(32) except: # By default, the exit code is 0, which typically indicates success. diff --git a/lib/galaxy/jobs/runners/cli.py b/lib/galaxy/jobs/runners/cli.py index f59648eeab6..13a617d8ec3 100644 --- a/lib/galaxy/jobs/runners/cli.py +++ b/lib/galaxy/jobs/runners/cli.py @@ -136,7 +136,7 @@ class ShellJobRunner( AsynchronousJobRunner ): ajs.job_id = external_job_id ajs.old_state = 'new' ajs.job_destination = job_destination - + # Add to our 'queue' of jobs to monitor self.monitor_queue.put( ajs ) diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py index 658303f8a90..9c7cf07ed08 100644 --- a/lib/galaxy/jobs/runners/condor.py +++ b/lib/galaxy/jobs/runners/condor.py @@ -41,7 +41,7 @@ default_query_classad = dict( class CondorJobState( AsynchronousJobState ): def __init__( self, **kwargs ): """ - Encapsulates state related to a job that is being run via the DRM and + Encapsulates state related to a job that is being run via the DRM and that we need to monitor. """ super( CondorJobState, self ).__init__( **kwargs ) @@ -227,7 +227,7 @@ class CondorJobRunner( AsynchronousJobRunner ): new_watched.append( cjs ) # Replace the watch list with the updated version self.watched = new_watched - + def stop_job( self, job ): """Attempts to delete a job from the DRM queue""" try: diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 007dd67d4e2..1a0a14df873 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -109,7 +109,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ): # command line has been added to the wrapper by prepare_job() command_line = job_wrapper.runner_command_line - + # get configured job destination job_destination = job_wrapper.job_destination @@ -182,7 +182,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ): ajs.job_id = external_job_id ajs.old_state = 'new' ajs.job_destination = job_destination - + # delete the job template self.ds.deleteJobTemplate( jt ) @@ -234,7 +234,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ): new_watched.append( ajs ) # Replace the watch list with the updated version self.watched = new_watched - + def stop_job( self, job ): """Attempts to delete a job from the DRM queue""" try: @@ -320,7 +320,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ): raise RuntimeError("External_runjob failed (exit code %s)\nChild process reported error:\n%s" % (str(exitcode), stderrdata)) if not stdoutdata.strip(): raise RuntimeError("External_runjob did return the job id: %s" % (stdoutdata)) - + # The expected output is a single line containing a single numeric value: # the DRMAA job-ID. If not the case, will throw an error. jobId = stdoutdata diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py index 421355adad7..7cf3c794f00 100644 --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -28,13 +28,13 @@ class LocalJobRunner( BaseJobRunner ): #create a local copy of os.environ to use as env for subprocess.Popen self._environ = os.environ.copy() - + # put lib into the PYTHONPATH for subprocesses if 'PYTHONPATH' in self._environ: self._environ['PYTHONPATH'] = '%s:%s' % ( self._environ['PYTHONPATH'], os.path.abspath( 'lib' ) ) else: self._environ['PYTHONPATH'] = os.path.abspath( 'lib' ) - + #Set TEMP if a valid temp value is not already set if not ( 'TMPDIR' in self._environ or 'TEMP' in self._environ or 'TMP' in self._environ ): self._environ[ 'TEMP' ] = tempfile.gettempdir() @@ -48,7 +48,7 @@ class LocalJobRunner( BaseJobRunner ): return stderr = stdout = '' - exit_code = 0 + exit_code = 0 # command line has been added to the wrapper by prepare_job() command_line = job_wrapper.runner_command_line @@ -59,9 +59,9 @@ class LocalJobRunner( BaseJobRunner ): log.debug( '(%s) executing: %s' % ( job_id, command_line ) ) stdout_file = tempfile.NamedTemporaryFile( suffix='_stdout', dir=job_wrapper.working_directory ) stderr_file = tempfile.NamedTemporaryFile( suffix='_stderr', dir=job_wrapper.working_directory ) - proc = subprocess.Popen( args = command_line, - shell = True, - cwd = job_wrapper.working_directory, + proc = subprocess.Popen( args = command_line, + shell = True, + cwd = job_wrapper.working_directory, stdout = stdout_file, stderr = stderr_file, env = self._environ, diff --git a/lib/galaxy/jobs/runners/pbs.py b/lib/galaxy/jobs/runners/pbs.py index 8c2a1866e04..462ab61da91 100644 --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -231,7 +231,7 @@ class PBSJobRunner( AsynchronousJobRunner ): command_line = job_wrapper.runner_command_line job_destination = job_wrapper.job_destination - + # Determine the job's PBS destination (server/queue) and options from the job destination definition pbs_queue_name = None pbs_server_name = self.default_pbs_server @@ -267,7 +267,7 @@ class PBSJobRunner( AsynchronousJobRunner ): ecfile = "%s/%s.ec" % (self.app.config.cluster_files_directory, job_wrapper.job_id) output_fnames = job_wrapper.get_output_fnames() - + # If an application server is set, we're staging if self.app.config.pbs_application_server: pbs_ofile = self.app.config.pbs_application_server + ':' + ofile @@ -329,7 +329,7 @@ class PBSJobRunner( AsynchronousJobRunner ): return # submit - # The job tag includes the job and the task identifier + # The job tag includes the job and the task identifier # (if a TaskWrapper was passed in): galaxy_job_id = job_wrapper.get_id_tag() log.debug("(%s) submitting file %s" % ( galaxy_job_id, job_file ) ) @@ -369,7 +369,7 @@ class PBSJobRunner( AsynchronousJobRunner ): job_state.old_state = 'N' job_state.running = False job_state.job_destination = job_destination - + # Add to our 'queue' of jobs to monitor self.monitor_queue.put( job_state ) @@ -450,7 +450,7 @@ class PBSJobRunner( AsynchronousJobRunner ): new_watched.append( pbs_job_state ) # Replace the watch list with the updated version self.watched = new_watched - + def check_all_jobs( self ): """ Returns a list of servers that failed to be contacted and a dict @@ -525,14 +525,14 @@ class PBSJobRunner( AsynchronousJobRunner ): ecfh = file(ecfile, "r") stdout = shrink_stream_by_size( ofh, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True ) stderr = shrink_stream_by_size( efh, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True ) - # This should be an 8-bit exit code, but read ahead anyway: + # This should be an 8-bit exit code, but read ahead anyway: exit_code_str = ecfh.read(32) except: stdout = '' stderr = 'Job output not returned by PBS: the output datasets were deleted while the job was running, the job was manually dequeued or there was a cluster error.' # By default, the exit code is 0, which usually indicates success # (although clearly some error happened). - exit_code_str = "" + exit_code_str = "" # Translate the exit code string to an integer; use 0 on failure. try: @@ -599,21 +599,21 @@ class PBSJobRunner( AsynchronousJobRunner ): pbs_server_name = self.__get_pbs_server( job.destination_params ) if pbs_server_name is None: log.debug("(%s) Job queued but no destination stored in job params, cannot delete" - % job_tag ) + % job_tag ) return c = pbs.pbs_connect( pbs_server_name ) if c <= 0: log.debug("(%s) Connection to PBS server for job delete failed" - % job_tag ) + % job_tag ) return pbs.pbs_deljob( c, job_id, '' ) - log.debug( "%s Removed from PBS queue before job completion" + log.debug( "%s Removed from PBS queue before job completion" % job_tag ) except: e = traceback.format_exc() - log.debug( "%s Unable to stop job: %s" % ( job_tag, e ) ) + log.debug( "%s Unable to stop job: %s" % ( job_tag, e ) ) finally: - # Cleanup: disconnect from the server. + # Cleanup: disconnect from the server. if ( None != c ): pbs.pbs_disconnect( c ) diff --git a/lib/galaxy/jobs/runners/tasks.py b/lib/galaxy/jobs/runners/tasks.py index 8887a716c86..6a9f0db06ce 100644 --- a/lib/galaxy/jobs/runners/tasks.py +++ b/lib/galaxy/jobs/runners/tasks.py @@ -39,7 +39,7 @@ class TaskedJobRunner( BaseJobRunner ): job_wrapper.set_job_destination(job_wrapper.job_destination) # This is the job's exit code, which will depend on the tasks' - # exit code. The overall job's exit code will be one of two values: + # exit code. The overall job's exit code will be one of two values: # o if the job is successful, then the last task scanned will be # used to determine the exit code. Note that this is not the same # thing as the last task to complete, which could be added later. @@ -59,8 +59,8 @@ class TaskedJobRunner( BaseJobRunner ): job_wrapper.fail("Job Splitting Failed, no match for '%s'" % parallelism) return tasks = splitter.do_split(job_wrapper) - # Not an option for now. Task objects don't *do* anything - # useful yet, but we'll want them tracked outside this thread + # Not an option for now. Task objects don't *do* anything + # useful yet, but we'll want them tracked outside this thread # to do anything. # if track_tasks_in_database: task_wrappers = [] @@ -76,21 +76,21 @@ class TaskedJobRunner( BaseJobRunner ): count_complete = 0 sleep_time = 1 # sleep/loop until no more progress can be made. That is when - # all tasks are one of { OK, ERROR, DELETED }. If a task + # all tasks are one of { OK, ERROR, DELETED }. If a task completed_states = [ model.Task.states.OK, \ model.Task.states.ERROR, \ model.Task.states.DELETED ] - # TODO: Should we report an error (and not merge outputs) if - # one of the subtasks errored out? Should we prevent any that + # TODO: Should we report an error (and not merge outputs) if + # one of the subtasks errored out? Should we prevent any that # are pending from being started in that case? - # SM: I'm - # If any task has an error, then we will stop all of them + # SM: I'm + # If any task has an error, then we will stop all of them # immediately. Tasks that are in the QUEUED state will be - # moved to the DELETED state. The task's runner should - # ignore tasks that are not in the QUEUED state. + # moved to the DELETED state. The task's runner should + # ignore tasks that are not in the QUEUED state. # Deleted tasks are not included right now. - # + # while tasks_complete is False: count_complete = 0 tasks_complete = True @@ -186,10 +186,10 @@ class TaskedJobRunner( BaseJobRunner ): # - If the task is queued, then mark it as deleted # so that the runner will not run it later. (It would # be great to remove stuff from a runner's queue before - # the runner picks it up, but that isn't possible in + # the runner picks it up, but that isn't possible in # most APIs.) - # - If the task is running, then tell the runner - # (via the dispatcher) to cancel the task. + # - If the task is running, then tell the runner + # (via the dispatcher) to cancel the task. # - Else the task is new or waiting (which should be # impossible) or in an error or deleted state already, # so skip it. @@ -201,19 +201,19 @@ class TaskedJobRunner( BaseJobRunner ): task = task_wrapper.get_task() task_state = task.get_state() if ( model.Task.states.QUEUED == task_state ): - log.debug( "_cancel_job for job %d: Task %d is not running; setting state to DELETED" + log.debug( "_cancel_job for job %d: Task %d is not running; setting state to DELETED" % ( job.get_id(), task.get_id() ) ) task_wrapper.change_state( task.states.DELETED ) # If a task failed, then the caller will have waited a few seconds # before recognizing the failure. In that time, a queued task could # have been picked up by a runner but not marked as running. - # So wait a few seconds so that we can eliminate such tasks once they + # So wait a few seconds so that we can eliminate such tasks once they # are running. sleep(5) for task_wrapper in task_wrappers: if ( model.Task.states.RUNNING == task_wrapper.get_state() ): task = task_wrapper.get_task() - log.debug( "_cancel_job for job %d: Stopping running task %d" + log.debug( "_cancel_job for job %d: Stopping running task %d" % ( job.get_id(), task.get_id() ) ) job_wrapper.app.job_manager.job_handler.dispatcher.stop( task ) diff --git a/lib/galaxy/jobs/splitters/multi.py b/lib/galaxy/jobs/splitters/multi.py index 0535328c0dd..4ad239a3909 100644 --- a/lib/galaxy/jobs/splitters/multi.py +++ b/lib/galaxy/jobs/splitters/multi.py @@ -136,7 +136,7 @@ def do_merge( job_wrapper, task_wrappers): output_dataset = outputs[output][0] output_type = output_dataset.datatype output_files = [os.path.join(dir,base_output_name) for dir in task_dirs] - # Just include those files f in the output list for which the + # Just include those files f in the output list for which the # file f exists; some files may not exist if a task fails. output_files = [ f for f in output_files if os.path.exists(f) ] if output_files: diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 6f953524276..7154b25c5a4 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -780,11 +780,11 @@ class History( object, DictifiableMixin, UsesAnnotations ): history_name = unicode(history_name, 'utf-8') return history_name - def dictify( self, view='collection', value_mapper = None ): + def dictify( self, view='collection', value_mapper = None ): # Get basic value. rval = super( History, self ).dictify( view=view, value_mapper=value_mapper ) - + # Add tags. tags_str_list = [] for tag in self.tags: @@ -793,7 +793,7 @@ class History( object, DictifiableMixin, UsesAnnotations ): tag_str += ":" + tag.user_value tags_str_list.append( tag_str ) rval[ 'tags' ] = tags_str_list - + return rval def set_from_dict( self, new_data ): @@ -1918,7 +1918,7 @@ class LibraryFolder( object, DictifiableMixin ): f = self while f.parent: l_path.insert(0, f.name) - f = f.parent + f = f.parent return l_path @property def parent_library( self ): diff --git a/lib/galaxy/model/custom_types.py b/lib/galaxy/model/custom_types.py index cd230a3e0bb..53296e6775c 100644 --- a/lib/galaxy/model/custom_types.py +++ b/lib/galaxy/model/custom_types.py @@ -89,7 +89,7 @@ class MetadataType( JSONType ): class UUIDType(TypeDecorator): """ Platform-independent UUID type. - + Based on http://docs.sqlalchemy.org/en/rel_0_8/core/types.html#backend-agnostic-guid-type Changed to remove sqlalchemy 0.8 specific code diff --git a/lib/galaxy/model/item_attrs.py b/lib/galaxy/model/item_attrs.py index d374fa40946..d7218409706 100644 --- a/lib/galaxy/model/item_attrs.py +++ b/lib/galaxy/model/item_attrs.py @@ -6,17 +6,17 @@ import logging log = logging.getLogger( __name__ ) class RuntimeException( Exception ): - pass + pass class UsesItemRatings: - """ + """ Mixin for getting and setting item ratings. - + Class makes two assumptions: (1) item-rating association table is named RatingAssocation - (2) item-rating association table has a column with a foreign key referencing + (2) item-rating association table has a column with a foreign key referencing item table that contains the item's id. - """ + """ def get_ave_item_rating_data( self, db_session, item, webapp_model=None ): """ Returns the average rating for an item.""" if webapp_model is None: @@ -33,7 +33,7 @@ class UsesItemRatings: ave_rating = 0 num_ratings = int( db_session.query( func.count( item_rating_assoc_class.rating ) ).filter( item_id_filter ).scalar() ) return ( ave_rating, num_ratings ) - + def rate_item( self, db_session, user, item, rating, webapp_model=None ): """ Rate an item. Return type is RatingAssociation. """ if webapp_model is None: @@ -53,7 +53,7 @@ class UsesItemRatings: item_rating.rating = rating db_session.flush() return item_rating - + def get_user_item_rating( self, db_session, user, item, webapp_model=None ): """ Returns user's rating for an item. Return type is RatingAssociation. """ if webapp_model is None: @@ -61,11 +61,11 @@ class UsesItemRatings: item_rating_assoc_class = self._get_item_rating_assoc_class( item, webapp_model=webapp_model ) if not item_rating_assoc_class: raise RuntimeException( "Item does not have ratings: %s" % item.__class__.__name__ ) - - # Query rating table by user and item id. + + # Query rating table by user and item id. item_id_filter = self._get_item_id_filter_str( item, item_rating_assoc_class ) return db_session.query( item_rating_assoc_class ).filter_by( user=user ).filter( item_id_filter ).first() - + def _get_item_rating_assoc_class( self, item, webapp_model=None ): """ Returns an item's item-rating association class. """ if webapp_model is None: @@ -82,10 +82,10 @@ class UsesItemRatings: if fk.references( item.table ): item_fk = fk break - + if not item_fk: raise RuntimeException( "Cannot find item id column in item-rating association table: %s, %s" % item_rating_assoc_class.__name__, item_rating_assoc_class.table.name ) - + # TODO: can we provide a better filter than a raw string? return "%s=%i" % ( item_fk.parent.name, item.id ) @@ -97,17 +97,17 @@ class UsesAnnotations: if annotation_obj: return galaxy.util.unicodify( annotation_obj.annotation ) return None - + def get_item_annotation_obj( self, db_session, user, item ): """ Returns a user's annotation object for an item. """ # Get annotation association class. annotation_assoc_class = self._get_annotation_assoc_class( item ) if not annotation_assoc_class: return None - + # Get annotation association object. annotation_assoc = db_session.query( annotation_assoc_class ).filter_by( user=user ) - + # TODO: use filtering like that in _get_item_id_filter_str() if item.__class__ == galaxy.model.History: annotation_assoc = annotation_assoc.filter_by( history=item ) @@ -122,7 +122,7 @@ class UsesAnnotations: elif item.__class__ == galaxy.model.Visualization: annotation_assoc = annotation_assoc.filter_by( visualization=item ) return annotation_assoc.first() - + def add_item_annotation( self, db_session, user, item, annotation ): """ Add or update an item's annotation; a user can only have a single annotation for an item. """ # Get/create annotation association object. @@ -143,7 +143,7 @@ class UsesAnnotations: if annotation_assoc: db_session.delete(annotation_assoc) db_session.flush() - + def copy_item_annotation( self, db_session, source_user, source_item, target_user, target_item ): """ Copy an annotation from a user/item source to a user/item target. """ if source_user and target_user: @@ -152,17 +152,17 @@ class UsesAnnotations: annotation = self.add_item_annotation( db_session, target_user, target_item, annotation_str ) return annotation return None - + def _get_annotation_assoc_class( self, item ): """ Returns an item's item-annotation association class. """ class_name = '%sAnnotationAssociation' % item.__class__.__name__ return getattr( galaxy.model, class_name, None ) class DictifiableMixin: - """ Mixin that enables objects to be converted to dictionaries. This is useful + """ Mixin that enables objects to be converted to dictionaries. This is useful when for sharing objects across boundaries, such as the API, tool scripts, and JavaScript code. """ - + def dictify( self, view='collection', value_mapper=None ): """ Return item dictionary. diff --git a/lib/galaxy/model/mapping_tests.py b/lib/galaxy/model/mapping_tests.py index af7b76b0506..951da3b4c47 100644 --- a/lib/galaxy/model/mapping_tests.py +++ b/lib/galaxy/model/mapping_tests.py @@ -29,7 +29,7 @@ class MappingTests( unittest.TestCase ): assert users[0].email == "james@foo.bar.baz" assert users[0].password == "password" assert len( users[0].histories ) == 1 - assert users[0].histories[0].name == "History 1" + assert users[0].histories[0].name == "History 1" hists = model.session.query( model.History ).all() assert hists[0].name == "History 1" assert hists[1].name == ( "H" * 255 ) @@ -47,7 +47,7 @@ class MappingTests( unittest.TestCase ): assert hists[0].name == "History 1" assert hists[1].name == "History 2b" # gvk TODO need to ad test for GalaxySessions, but not yet sure what they should look like. - + def get_suite(): suite = unittest.TestSuite() suite.addTest( MappingTests( "test_basic" ) ) diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py index a5c7e603d43..40223f15295 100644 --- a/lib/galaxy/model/migrate/check.py +++ b/lib/galaxy/model/migrate/check.py @@ -25,7 +25,7 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=N Check that the database is use-able, possibly creating it if empty (this is the only time we automatically create tables, otherwise we force the user to do it using the management script so they can create backups). - + 1) Empty database --> initialize with latest version and return 2) Database older than migration support --> fail and require manual update 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update) @@ -104,7 +104,7 @@ def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=N % ( db_schema.version, migrate_repository.versions.latest, config_arg ) ) else: log.info( "At database version %d" % db_schema.version ) - + def migrate_to_current_version( engine, schema ): # Changes to get to current version changeset = schema.changeset( None ) diff --git a/lib/galaxy/model/orm/logging_connection_proxy.py b/lib/galaxy/model/orm/logging_connection_proxy.py index ecc50a0f649..5ed981f0868 100644 --- a/lib/galaxy/model/orm/logging_connection_proxy.py +++ b/lib/galaxy/model/orm/logging_connection_proxy.py @@ -42,7 +42,7 @@ class TraceLoggerProxy(ConnectionProxy): start = time.clock() rval = execute(cursor, statement, parameters, context) duration = time.clock() - start - self.trace_logger.log( "sqlalchemy_query", - message="Query executed", statement=statement, parameters=parameters, - executemany=executemany, duration=duration ) + self.trace_logger.log( "sqlalchemy_query", + message="Query executed", statement=statement, parameters=parameters, + executemany=executemany, duration=duration ) return rval \ No newline at end of file diff --git a/lib/galaxy/model/search.py b/lib/galaxy/model/search.py index d302b2a344a..376c3737756 100644 --- a/lib/galaxy/model/search.py +++ b/lib/galaxy/model/search.py @@ -31,10 +31,10 @@ from galaxy import eggs eggs.require("Parsley") import parsley -from galaxy.model import (HistoryDatasetAssociation, LibraryDatasetDatasetAssociation, -History, Library, LibraryFolder, LibraryDataset,StoredWorkflowTagAssociation, +from galaxy.model import (HistoryDatasetAssociation, LibraryDatasetDatasetAssociation, +History, Library, LibraryFolder, LibraryDataset,StoredWorkflowTagAssociation, StoredWorkflow, HistoryTagAssociation,HistoryDatasetAssociationTagAssociation, -ExtendedMetadata, ExtendedMetadataIndex, HistoryAnnotationAssociation, Job, JobParameter, +ExtendedMetadata, ExtendedMetadataIndex, HistoryAnnotationAssociation, Job, JobParameter, JobToInputDatasetAssociation, JobToOutputDatasetAssociation, ToolVersion) from galaxy.util.json import to_json_string diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index b68e1a4477b..98fdd7c5f05 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -331,7 +331,7 @@ class DiskObjectStore(ObjectStore): def update_from_file(self, obj, file_name=None, create=False, **kwargs): """ `create` parameter is not used in this implementation """ preserve_symlinks = kwargs.pop( 'preserve_symlinks', False ) - #FIXME: symlinks and the object store model may not play well together + #FIXME: symlinks and the object store model may not play well together #these should be handled better, e.g. registering the symlink'd file as an object if create: self.create(obj, **kwargs) diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 06f84b35fef..e482ae36bf0 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -14,7 +14,7 @@ from galaxy import util from galaxy.jobs import Sleeper from galaxy.model import directory_hash_id from galaxy.objectstore import ObjectStore, convert_bytes -from galaxy.exceptions import ObjectNotFound, ObjectInvalid +from galaxy.exceptions import ObjectNotFound import multiprocessing from galaxy.objectstore.s3_multipart_upload import multipart_upload diff --git a/lib/galaxy/objectstore/s3_multipart_upload.py b/lib/galaxy/objectstore/s3_multipart_upload.py index 79b39ad3182..a9f14acac96 100644 --- a/lib/galaxy/objectstore/s3_multipart_upload.py +++ b/lib/galaxy/objectstore/s3_multipart_upload.py @@ -76,7 +76,7 @@ def multipart_upload(bucket, s3_key_name, tarball, mb_size, use_rr=True): @contextlib.contextmanager def multimap(cores=None): """Provide multiprocessing imap like function. - + The context manager handles setting up the pool, worked around interrupt issues and terminating the pool on completion. """ diff --git a/lib/galaxy/quota/__init__.py b/lib/galaxy/quota/__init__.py index 99f99bea19f..ad6890564d5 100644 --- a/lib/galaxy/quota/__init__.py +++ b/lib/galaxy/quota/__init__.py @@ -120,7 +120,7 @@ class QuotaAgent( NoQuotaAgent ): dqa = self.model.DefaultQuotaAssociation( default_type, quota ) self.sa_session.add( dqa ) self.sa_session.flush() - + def get_percent( self, trans=None, user=False, history=False, usage=False, quota=False ): """ Return the percentage of any storage quota applicable to the user/transaction. diff --git a/lib/galaxy/sample_tracking/data_transfer.py b/lib/galaxy/sample_tracking/data_transfer.py index 31c2683e227..eb5598af95d 100644 --- a/lib/galaxy/sample_tracking/data_transfer.py +++ b/lib/galaxy/sample_tracking/data_transfer.py @@ -11,20 +11,20 @@ class ScpDataTransferFactory( DataTransferFactory ): pass def parse( self, config_file, elem ): self.config = {} - # TODO: The 'automatic_transfer' setting is for future use. If set to True, we will need to + # TODO: The 'automatic_transfer' setting is for future use. If set to True, we will need to # ensure the sample has an associated destination data library before it moves to a certain state # ( e.g., Run started ). self.config[ 'automatic_transfer' ] = elem.get( 'automatic_transfer' ) - self.config[ 'host' ] = elem.get( 'host' ) + self.config[ 'host' ] = elem.get( 'host' ) self.config[ 'user_name' ] = elem.get( 'user_name' ) - self.config[ 'password' ] = elem.get( 'password' ) + self.config[ 'password' ] = elem.get( 'password' ) self.config[ 'data_location' ] = elem.get( 'data_location' ) # 'rename_dataset' is optional and it may not be defined in all external types # It is only used is AB SOLiD external service type for now rename_dataset = elem.get( 'rename_dataset', None ) if rename_dataset: self.config['rename_dataset'] = rename_dataset - # Validate + # Validate for name, value in self.config.items(): assert value, "'%s' attribute missing in 'data_transfer' element of type 'scp' in external_service_type xml config file: '%s'." % ( name, config_file ) @@ -35,7 +35,7 @@ class HttpDataTransferFactory( DataTransferFactory ): def parse( self, config_file, elem ): self.config = {} self.config[ 'automatic_transfer' ] = elem.get( 'automatic_transfer' ) - # Validate + # Validate for name, value in self.config.items(): assert value, "'%s' attribute missing in 'data_transfer' element of type 'http' in external_service_type xml config file: '%s'." % ( name, config_file ) diff --git a/lib/galaxy/sample_tracking/external_service_types.py b/lib/galaxy/sample_tracking/external_service_types.py index 09cc150d8a2..cb4c6e3262d 100644 --- a/lib/galaxy/sample_tracking/external_service_types.py +++ b/lib/galaxy/sample_tracking/external_service_types.py @@ -62,16 +62,16 @@ class ExternalServiceType( object ): self.visible = visible root.clear() def parse( self, root ): - # Get the name + # Get the name self.name = root.get( "name" ) - if not self.name: + if not self.name: raise Exception, "Missing external_service_type 'name'" - # Get the UNIQUE id for the tool + # Get the UNIQUE id for the tool self.id = root.get( "id" ) - if not self.id: + if not self.id: raise Exception, "Missing external_service_type 'id'" self.config_version = root.get( "version" ) - if not self.config_version: + if not self.config_version: self.config_version = '1.0.0' self.description = util.xml_text(root, "description") self.version = util.xml_text( root.find( "version" ) ) diff --git a/lib/galaxy/sample_tracking/request_types.py b/lib/galaxy/sample_tracking/request_types.py index 879b99e4bc8..975eb9201e7 100644 --- a/lib/galaxy/sample_tracking/request_types.py +++ b/lib/galaxy/sample_tracking/request_types.py @@ -13,7 +13,7 @@ class RequestTypeFactory( object ): self.rename_dataset_options = rename_dataset_options def new( self, name, request_form, sample_form, external_service, description=None, sample_states = None ): """Return new RequestType.""" - assert name, 'RequestType requires a name' + assert name, 'RequestType requires a name' return RequestType( name=name, desc=description, request_form=request_form, sample_form=sample_form, external_service=external_service ) def from_elem( self, elem, request_form, sample_form, external_service ): """Return RequestType created from an xml string.""" diff --git a/lib/galaxy/security/__init__.py b/lib/galaxy/security/__init__.py index ee1ee67c72b..be02177f856 100644 --- a/lib/galaxy/security/__init__.py +++ b/lib/galaxy/security/__init__.py @@ -1,1316 +1,1316 @@ -""" -Galaxy Security - -""" -import logging, socket, operator -from datetime import datetime, timedelta -from galaxy.util.bunch import Bunch -from galaxy.util import listify -from galaxy.model.orm import * - -log = logging.getLogger(__name__) - -class Action( object ): - def __init__( self, action, description, model ): - self.action = action - self.description = description - self.model = model - -class RBACAgent: - """Class that handles galaxy security""" - permitted_actions = Bunch( - DATASET_MANAGE_PERMISSIONS = Action( "manage permissions", "Users having associated role can manage the roles associated with permissions on this dataset", "grant" ), - DATASET_ACCESS = Action( "access", "Users having associated role can import this dataset into their history for analysis", "restrict" ), - LIBRARY_ACCESS = Action( "access library", "Restrict access to this library to only users having associated role", "restrict" ), - LIBRARY_ADD = Action( "add library item", "Users having associated role can add library items to this library item", "grant" ), - LIBRARY_MODIFY = Action( "modify library item", "Users having associated role can modify this library item", "grant" ), - LIBRARY_MANAGE = Action( "manage library permissions", "Users having associated role can manage roles associated with permissions on this library item", "grant" ), - # Request type permissions - REQUEST_TYPE_ACCESS = Action( "access request_type", "Restrict access to this request type to only users having associated role", "restrict" ) - - ) - def get_action( self, name, default=None ): - """Get a permitted action by its dict key or action name""" - for k, v in self.permitted_actions.items(): - if k == name or v.action == name: - return v - return default - def get_actions( self ): - """Get all permitted actions as a list of Action objects""" - return self.permitted_actions.__dict__.values() - def get_item_actions( self, action, item ): - raise 'No valid method of retrieving action (%s) for item %s.' % ( action, item ) - def guess_derived_permissions_for_datasets( self, datasets = [] ): - raise "Unimplemented Method" - def can_access_dataset( self, roles, dataset ): - raise "Unimplemented Method" - def can_manage_dataset( self, roles, dataset ): - raise "Unimplemented Method" - def can_access_library( self, roles, library ): - raise "Unimplemented Method" - def can_add_library_item( self, roles, item ): - raise "Unimplemented Method" - def can_modify_library_item( self, roles, item ): - raise "Unimplemented Method" - def can_manage_library_item( self, roles, item ): - raise "Unimplemented Method" - def associate_components( self, **kwd ): - raise 'No valid method of associating provided components: %s' % kwd - def create_private_user_role( self, user ): - raise "Unimplemented Method" - def get_private_user_role( self, user ): - raise "Unimplemented Method" - def get_accessible_request_types( self, trans, user ): - raise "Unimplemented Method" - def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False ): - raise "Unimplemented Method" - def history_set_default_permissions( self, history, permissions=None, dataset=False, bypass_manage_permission=False ): - raise "Unimplemented Method" - def set_all_dataset_permissions( self, dataset, permissions ): - raise "Unimplemented Method" - def set_dataset_permission( self, dataset, permission ): - raise "Unimplemented Method" - def set_all_library_permissions( self, trans, dataset, permissions ): - raise "Unimplemented Method" - def library_is_public( self, library ): - raise "Unimplemented Method" - def make_library_public( self, library ): - raise "Unimplemented Method" - def get_accessible_libraries( self, trans, user ): - raise "Unimplemented Method" - def get_permitted_libraries( self, trans, user, actions ): - raise "Unimplemented Method" - def folder_is_public( self, library ): - raise "Unimplemented Method" - def make_folder_public( self, folder, count=0 ): - raise "Unimplemented Method" - def dataset_is_public( self, dataset ): - raise "Unimplemented Method" - def make_dataset_public( self, dataset ): - raise "Unimplemented Method" - def get_permissions( self, library_dataset ): - raise "Unimplemented Method" - def get_all_roles( self, trans, cntrller ): - raise "Unimplemented Method" - def get_legitimate_roles( self, trans, item, cntrller ): - raise "Unimplemented Method" - def derive_roles_from_access( self, trans, item_id, cntrller, library=False, **kwd ): - raise "Unimplemented Method" - def get_component_associations( self, **kwd ): - raise "Unimplemented Method" - def components_are_associated( self, **kwd ): - return bool( self.get_component_associations( **kwd ) ) - def convert_permitted_action_strings( self, permitted_action_strings ): - """ - When getting permitted actions from an untrusted source like a - form, ensure that they match our actual permitted actions. - """ - return filter( lambda x: x is not None, [ self.permitted_actions.get( action_string ) for action_string in permitted_action_strings ] ) - -class GalaxyRBACAgent( RBACAgent ): - def __init__( self, model, permitted_actions=None ): - self.model = model - if permitted_actions: - self.permitted_actions = permitted_actions - # List of "library_item" objects and their associated permissions and info template objects - self.library_item_assocs = ( - ( self.model.Library, self.model.LibraryPermissions ), - ( self.model.LibraryFolder, self.model.LibraryFolderPermissions ), - ( self.model.LibraryDataset, self.model.LibraryDatasetPermissions ), - ( self.model.LibraryDatasetDatasetAssociation, self.model.LibraryDatasetDatasetAssociationPermissions ) ) - @property - def sa_session( self ): - """Returns a SQLAlchemy session""" - return self.model.context - def sort_by_attr( self, seq, attr ): - """ - Sort the sequence of objects by object's attribute - Arguments: - seq - the list or any sequence (including immutable one) of objects to sort. - attr - the name of attribute to sort by - """ - # Use the "Schwartzian transform" - # Create the auxiliary list of tuples where every i-th tuple has form - # (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not - # only to provide stable sorting, but mainly to eliminate comparison of objects - # (which can be expensive or prohibited) in case of equal attribute values. - intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq ) - intermed.sort() - return map( operator.getitem, intermed, ( -1, ) * len( intermed ) ) - def get_all_roles( self, trans, cntrller ): - admin_controller = cntrller in [ 'library_admin' ] - roles = set() - if not trans.user: - return trans.sa_session.query( trans.app.model.Role ) \ - .filter( and_( self.model.Role.table.c.deleted==False, - self.model.Role.table.c.type != self.model.Role.types.PRIVATE, - self.model.Role.table.c.type != self.model.Role.types.SHARING ) ) \ - .order_by( self.model.Role.table.c.name ) - if admin_controller: - # The library is public and the user is an admin, so all roles are legitimate - for role in trans.sa_session.query( trans.app.model.Role ) \ - .filter( self.model.Role.table.c.deleted==False ) \ - .order_by( self.model.Role.table.c.name ): - roles.add( role ) - else: - # Add the current user's private role - roles.add( self.get_private_user_role( trans.user ) ) - # Add the current user's sharing roles - for role in self.get_sharing_roles( trans.user ): - roles.add( role ) - # Add all remaining non-private, non-sharing roles - for role in trans.sa_session.query( trans.app.model.Role ) \ - .filter( and_( self.model.Role.table.c.deleted==False, - self.model.Role.table.c.type != self.model.Role.types.PRIVATE, - self.model.Role.table.c.type != self.model.Role.types.SHARING ) ) \ - .order_by( self.model.Role.table.c.name ): - roles.add( role ) - return self.sort_by_attr( [ role for role in roles ], 'name' ) - def get_legitimate_roles( self, trans, item, cntrller ): - """ - Return a sorted list of legitimate roles that can be associated with a permission on - item where item is a Library or a Dataset. The cntrller param is the controller from - which the request is sent. We cannot use trans.user_is_admin() because the controller is - what is important since admin users do not necessarily have permission to do things - on items outside of the admin view. - - If cntrller is from the admin side ( e.g., library_admin ): - - - if item is public, all roles, including private roles, are legitimate. - - if item is restricted, legitimate roles are derived from the users and groups associated - with each role that is associated with the access permission ( i.e., DATASET_MANAGE_PERMISSIONS or - LIBRARY_MANAGE ) on item. Legitimate roles will include private roles. - - If cntrller is not from the admin side ( e.g., root, library ): - - - if item is public, all non-private roles, except for the current user's private role, - are legitimate. - - if item is restricted, legitimate roles are derived from the users and groups associated - with each role that is associated with the access permission on item. Private roles, except - for the current user's private role, will be excluded. - """ - admin_controller = cntrller in [ 'library_admin' ] - roles = set() - if ( isinstance( item, self.model.Library ) and self.library_is_public( item ) ) or \ - ( isinstance( item, self.model.Dataset ) and self.dataset_is_public( item ) ): - return self.get_all_roles( trans, cntrller ) - # If item has roles associated with the access permission, we need to start with them. - access_roles = item.get_access_roles( trans ) - for role in access_roles: - if admin_controller or self.ok_to_display( trans.user, role ): - roles.add( role ) - # Each role potentially has users. We need to find all roles that each of those users have. - for ura in role.users: - user = ura.user - for ura2 in user.roles: - if admin_controller or self.ok_to_display( trans.user, ura2.role ): - roles.add( ura2.role ) - # Each role also potentially has groups which, in turn, have members ( users ). We need to - # find all roles that each group's members have. - for gra in role.groups: - group = gra.group - for uga in group.users: - user = uga.user - for ura in user.roles: - if admin_controller or self.ok_to_display( trans.user, ura.role ): - roles.add( ura.role ) - return self.sort_by_attr( [ role for role in roles ], 'name' ) - def ok_to_display( self, user, role ): - """ - Method for checking if: - - a role is private and is the current user's private role - - a role is a sharing role and belongs to the current user - """ - if user: - if role.type == self.model.Role.types.PRIVATE: - return role == self.get_private_user_role( user ) - if role.type == self.model.Role.types.SHARING: - return role in self.get_sharing_roles( user ) - # If role.type is neither private nor sharing, it's ok to display - return True - return role.type != self.model.Role.types.PRIVATE and role.type != self.model.Role.types.SHARING - - def allow_action( self, roles, action, item ): - """ - Method for checking a permission for the current user ( based on roles ) to perform a - specific action on an item, which must be one of: - Dataset, Library, LibraryFolder, LibraryDataset, LibraryDatasetDatasetAssociation - """ - # SM: Note that calling get_item_actions will emit a query. - item_actions = self.get_item_actions( action, item ) - - if not item_actions: - return action.model == 'restrict' - ret_val = False - # For DATASET_ACCESS only, user must have ALL associated roles - if action == self.permitted_actions.DATASET_ACCESS: - for item_action in item_actions: - if item_action.role not in roles: - break - else: - ret_val = True - # For remaining actions, user must have any associated role - else: - for item_action in item_actions: - if item_action.role in roles: - ret_val = True - break - return ret_val - - - def get_actions_for_items( self, trans, action, permission_items ): - # TODO: Rename this; it's a replacement for get_item_actions, but it - # doesn't represent what it's really doing, which is confusing. - # TODO: Make this work for other classes besides lib_datasets. - # That should be as easy as checking the type and writing a query for each; - # we're avoiding using the SQLAlchemy backrefs because they can cause lots - # of queries to be generated. - # - # Originally, get_item_actions did: - # return [ permission for permission in item.actions if permission.action == action.action ] - # The "item" can be just about anything with permissions, and referencing - # item.actions causes the item's permissions to be retrieved. - # This method will retrieve all permissions for all "items" and only - # return the permissions associated with that given action. - # We initialize the permissions list to be empty; we will return an - # empty list by default. - # - # If the dataset id has no corresponding action in its permissions, - # then the returned permissions will not carry an entry for the dataset. - ret_permissions = {} - if ( len( permission_items ) > 0 ): - # SM: NB: LibraryDatasets became Datasets for some odd reason. - if ( isinstance( permission_items[0], trans.model.LibraryDataset ) ): - ids = [ item.library_dataset_id for item in permission_items ] - permissions = trans.sa_session.query( trans.model.LibraryDatasetPermissions ) \ - .filter( and_( trans.model.LibraryDatasetPermissions.library_dataset_id.in_( ids ), - trans.model.LibraryDatasetPermissions.action == action.action ) ) \ - .all() - - # Massage the return data. We will return a list of permissions - # for each library dataset. So we initialize the return list to - # have an empty list for each dataset. Then each permission is - # appended to the right lib dataset. - # TODO: Consider eliminating the initialization and just return - # empty values for each library dataset id. - for item in permission_items: - ret_permissions[ item.library_dataset_id ] = [] - for permission in permissions: - ret_permissions[ permission.library_dataset_id ].append( permission ) - elif ( isinstance( permission_items[0], trans.model.Dataset ) ): - ids = [ item.id for item in permission_items ] - permissions = trans.sa_session.query( trans.model.DatasetPermissions ) \ - .filter( and_( trans.model.DatasetPermissions.dataset_id.in_( ids ), - trans.model.DatasetPermissions.action == action.action ) ) \ - .all() - - # Massage the return data. We will return a list of permissions - # for each library dataset. So we initialize the return list to - # have an empty list for each dataset. Then each permission is - # appended to the right lib dataset. - # TODO: Consider eliminating the initialization and just return - # empty values for each library dataset id. - for item in permission_items: - ret_permissions[ item.id ] = [] - for permission in permissions: - ret_permissions[ permission.dataset_id ].append( permission ) - - # Test that we get the same response from get_item_actions each item: - test_code = False - if test_code: - try: - log.debug( "get_actions_for_items: Test start" ) - for item in permission_items: - base_result = self.get_item_actions( action, item ) - new_result = ret_permissions[ item.library_dataset_id ] - # For now, just test against LibraryDatasetIds; other classes - # are not tested yet. - if len( base_result ) == len( new_result ): - common_result = set(base_result).intersection( new_result ) - if len( common_result ) == len( base_result ): - log.debug( "Match on permissions for id %d" % - item.library_dataset_id ) - # TODO: Fix this failure message: - else: - log.debug( "Error: dataset %d; originally: %s; now: %s" - % ( item.library_dataset_id, - base_result, new_result ) ) - else: - log.debug( "Error: dataset %d: had %d entries, now %d entries" - % ( item.library_dataset_id, len( base_result ), - len( new_result ) ) ) - log.debug( "get_actions_for_items: Test end" ) - except Exception, e: - log.debug( "Exception in test code: %s" % e ) - - return ret_permissions - - - def allow_action_on_libitems( self, trans, user_roles, action, items ): - """ - This should be the equivalent of allow_action defined on multiple items. - It is meant to specifically replace allow_action for multiple - LibraryDatasets, but it could be reproduced or modified for - allow_action's permitted classes - Dataset, Library, LibraryFolder, and - LDDAs. - """ - all_items_actions = self.get_actions_for_items( trans, action, items ) - ret_allow_action = {} - - # Change item to lib_dataset or vice-versa. - for item in items: - if all_items_actions.has_key( item.id ): - item_actions = all_items_actions[ item.id ] - - if self.permitted_actions.DATASET_ACCESS == action: - ret_allow_action[ item.id ] = True - for item_action in item_actions: - if item_action.role not in user_roles: - ret_allow_action[ item.id ] = False - break - - # Else look for just one dataset role to be in the list of - # acceptable user roles: - else: - ret_allow_action[ item.id ] = False - for item_action in item_actions: - if item_action.role in user_roles: - ret_allow_action[ item.id ] = True - break - - else: - if 'restrict' == action.model: - ret_allow_action[ item.id ] = True - else: - ret_allow_action[ item.id ] = False - - # Test it: the result for each dataset should match the result for - # allow_action: - test_code = False - if test_code: - log.debug( "allow_action_for_items: test start" ) - for item in items: - orig_value = self.allow_action( user_roles, action, item ) - if orig_value == ret_allow_action[ item.id ]: - log.debug( "Item %d: success" % item.id ) - else: - log.debug( "Item %d: fail: original: %s; new: %s" - % ( item.id, orig_value, ret_allow_action[ item.id ] ) ) - log.debug( "allow_action_for_items: test end" ) - return ret_allow_action - - - # DELETEME: SM: DO NOT TOUCH! This actually works. - def dataset_access_mapping( self, trans, user_roles, datasets ): - ''' - For the given list of datasets, return a mapping of the datasets' ids - to whether they can be accessed by the user or not. The datasets input - is expected to be a simple list of Dataset objects. - ''' - datasets_public_map = self.datasets_are_public( trans, datasets ) - datasets_allow_action_map = self.allow_action_on_libitems( trans, user_roles, self.permitted_actions.DATASET_ACCESS, datasets ) - can_access = {} - for dataset in datasets: - can_access[ dataset.id ] = datasets_public_map[ dataset.id ] or datasets_allow_action_map[ dataset.id ] - return can_access - - def dataset_permission_map_for_access( self, trans, user_roles, libitems ): - ''' - For a given list of library items (e.g., Datasets), return a map of the - datasets' ids to whether they can have permission to use that action - (e.g., "access" or "modify") on the dataset. The libitems input is - expected to be a simple list of library items, such as Datasets or - LibraryDatasets. - NB: This is currently only usable for Datasets; it was intended to - be used for any library item. - ''' - # Map the library items to whether they are publicly accessible or not. - # Then determine what actions are allowed on the item (in case it's not - # public). Finally, the item is accessible if it's publicly available - # or the right permissions are enabled. - # TODO: This only works for Datasets; other code is using X_is_public, - # so this will have to be rewritten to support other items. - libitems_public_map = self.datasets_are_public( trans, libitems ) - libitems_allow_action_map = self.allow_action_on_libitems( - trans, user_roles, self.permitted_actions.DATASET_ACCESS, libitems ) - can_access = {} - for libitem in libitems: - can_access[ libitem.id ] = libitems_public_map[ libitem.id ] or libitems_allow_action_map[ libitem.id ] - return can_access - - def item_permission_map_for_modify( self, trans, user_roles, libitems ): - return self.allow_action_on_libitems( - trans, user_roles, self.permitted_actions.LIBRARY_MODIFY, libitems ) - - def item_permission_map_for_manage( self, trans, user_roles, libitems ): - return self.allow_action_on_libitems( - trans, user_roles, self.permitted_actions.LIBRARY_MANAGE, libitems ) - - def item_permission_map_for_add( self, trans, user_roles, libitems ): - return self.allow_action_on_libitems( - trans, user_roles, self.permitted_actions.LIBRARY_ADD, libitems ) - - def can_access_dataset( self, user_roles, dataset ): - # SM: dataset_is_public will access dataset.actions, which is a - # backref that causes a query to be made to DatasetPermissions - retval = self.dataset_is_public( dataset ) or self.allow_action( user_roles, self.permitted_actions.DATASET_ACCESS, dataset ) - return retval - - def can_manage_dataset( self, roles, dataset ): - return self.allow_action( roles, self.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset ) - def can_access_library( self, roles, library ): - return self.library_is_public( library ) or self.allow_action( roles, self.permitted_actions.LIBRARY_ACCESS, library ) - def get_accessible_libraries( self, trans, user ): - """Return all data libraries that the received user can access""" - accessible_libraries = [] - current_user_role_ids = [ role.id for role in user.all_roles() ] - library_access_action = self.permitted_actions.LIBRARY_ACCESS.action - restricted_library_ids = [ lp.library_id for lp in trans.sa_session.query( trans.model.LibraryPermissions ) \ - .filter( trans.model.LibraryPermissions.table.c.action == library_access_action ) \ - .distinct() ] - accessible_restricted_library_ids = [ lp.library_id for lp in trans.sa_session.query( trans.model.LibraryPermissions ) \ - .filter( and_( trans.model.LibraryPermissions.table.c.action == library_access_action, - trans.model.LibraryPermissions.table.c.role_id.in_( current_user_role_ids ) ) ) ] - # Filter to get libraries accessible by the current user. Get both - # public libraries and restricted libraries accessible by the current user. - for library in trans.sa_session.query( trans.model.Library ) \ - .filter( and_( trans.model.Library.table.c.deleted == False, - ( or_( not_( trans.model.Library.table.c.id.in_( restricted_library_ids ) ), - trans.model.Library.table.c.id.in_( accessible_restricted_library_ids ) ) ) ) ) \ - .order_by( trans.app.model.Library.name ): - accessible_libraries.append( library ) - return accessible_libraries - def has_accessible_folders( self, trans, folder, user, roles, search_downward=True ): - if self.has_accessible_library_datasets( trans, folder, user, roles, search_downward=search_downward ) or \ - self.can_add_library_item( roles, folder ) or \ - self.can_modify_library_item( roles, folder ) or \ - self.can_manage_library_item( roles, folder ): - return True - if search_downward: - for folder in folder.active_folders: - return self.has_accessible_folders( trans, folder, user, roles, search_downward=search_downward ) - return False - def has_accessible_library_datasets( self, trans, folder, user, roles, search_downward=True ): - for library_dataset in trans.sa_session.query( trans.model.LibraryDataset ) \ - .filter( and_( trans.model.LibraryDataset.table.c.deleted == False, - trans.app.model.LibraryDataset.table.c.folder_id==folder.id ) ): - if self.can_access_library_item( roles, library_dataset, user ): - return True - if search_downward: - return self.__active_folders_have_accessible_library_datasets( trans, folder, user, roles ) - return False - def __active_folders_have_accessible_library_datasets( self, trans, folder, user, roles ): - for active_folder in folder.active_folders: - if self.has_accessible_library_datasets( trans, active_folder, user, roles ): - return True - return False - def can_access_library_item( self, roles, item, user ): - if type( item ) == self.model.Library: - return self.can_access_library( roles, item ) - elif type( item ) == self.model.LibraryFolder: - return self.can_access_library( roles, item.parent_library ) and self.check_folder_contents( user, roles, item )[0] - elif type( item ) == self.model.LibraryDataset: - return self.can_access_library( roles, item.folder.parent_library ) and self.can_access_dataset( roles, item.library_dataset_dataset_association.dataset ) - elif type( item ) == self.model.LibraryDatasetDatasetAssociation: - return self.can_access_library( roles, item.library_dataset.folder.parent_library ) and self.can_access_dataset( roles, item.dataset ) - else: - log.warning( 'Unknown library item type: %s' % type ( item ) ) - return False - def can_add_library_item( self, roles, item ): - return self.allow_action( roles, self.permitted_actions.LIBRARY_ADD, item ) - def can_modify_library_item( self, roles, item ): - return self.allow_action( roles, self.permitted_actions.LIBRARY_MODIFY, item ) - def can_manage_library_item( self, roles, item ): - return self.allow_action( roles, self.permitted_actions.LIBRARY_MANAGE, item ) - - def get_item_actions( self, action, item ): - # item must be one of: Dataset, Library, LibraryFolder, LibraryDataset, LibraryDatasetDatasetAssociation - # SM: Accessing item.actions emits a query to Library_Dataset_Permissions - # if the item is a LibraryDataset: - # TODO: Pass in the item's actions - the item isn't needed - return [ permission for permission in item.actions if permission.action == action.action ] - - def guess_derived_permissions_for_datasets( self, datasets=[] ): - """Returns a dict of { action : [ role, role, ... ] } for the output dataset based upon provided datasets""" - perms = {} - for dataset in datasets: - if not isinstance( dataset, self.model.Dataset ): - dataset = dataset.dataset - these_perms = {} - # initialize blank perms - for action in self.get_actions(): - these_perms[ action ] = [] - # collect this dataset's perms - these_perms = self.get_permissions( dataset ) - # join or intersect this dataset's permissions with others - for action, roles in these_perms.items(): - if action not in perms.keys(): - perms[ action ] = roles - else: - if action.model == 'grant': - # intersect existing roles with new roles - perms[ action ] = filter( lambda x: x in perms[ action ], roles ) - elif action.model == 'restrict': - # join existing roles with new roles - perms[ action ].extend( filter( lambda x: x not in perms[ action ], roles ) ) - return perms - def associate_components( self, **kwd ): - if 'user' in kwd: - if 'group' in kwd: - return self.associate_user_group( kwd['user'], kwd['group'] ) - elif 'role' in kwd: - return self.associate_user_role( kwd['user'], kwd['role'] ) - elif 'role' in kwd: - if 'group' in kwd: - return self.associate_group_role( kwd['group'], kwd['role'] ) - if 'action' in kwd: - if 'dataset' in kwd and 'role' in kwd: - return self.associate_action_dataset_role( kwd['action'], kwd['dataset'], kwd['role'] ) - raise 'No valid method of associating provided components: %s' % kwd - def associate_user_group( self, user, group ): - assoc = self.model.UserGroupAssociation( user, group ) - self.sa_session.add( assoc ) - self.sa_session.flush() - return assoc - def associate_user_role( self, user, role ): - assoc = self.model.UserRoleAssociation( user, role ) - self.sa_session.add( assoc ) - self.sa_session.flush() - return assoc - def associate_group_role( self, group, role ): - assoc = self.model.GroupRoleAssociation( group, role ) - self.sa_session.add( assoc ) - self.sa_session.flush() - return assoc - def associate_action_dataset_role( self, action, dataset, role ): - assoc = self.model.DatasetPermissions( action, dataset, role ) - self.sa_session.add( assoc ) - self.sa_session.flush() - return assoc - def create_private_user_role( self, user ): - # Create private role - role = self.model.Role( name=user.email, description='Private Role for ' + user.email, type=self.model.Role.types.PRIVATE ) - self.sa_session.add( role ) - self.sa_session.flush() - # Add user to role - self.associate_components( role=role, user=user ) - return role - def get_private_user_role( self, user, auto_create=False ): - role = self.sa_session.query( self.model.Role ) \ - .filter( and_( self.model.Role.table.c.name == user.email, - self.model.Role.table.c.type == self.model.Role.types.PRIVATE ) ) \ - .first() - if not role: - if auto_create: - return self.create_private_user_role( user ) - else: - return None - return role - def get_sharing_roles( self, user ): - return self.sa_session.query( self.model.Role ) \ - .filter( and_( ( self.model.Role.table.c.name ).like( "Sharing role for: %" + user.email + "%" ), - self.model.Role.table.c.type == self.model.Role.types.SHARING ) ) - def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False, bypass_manage_permission=False, default_access_private = False ): - # bypass_manage_permission is used to change permissions of datasets in a userless history when logging in - flush_needed = False - if user is None: - return None - if not permissions: - #default permissions - permissions = { self.permitted_actions.DATASET_MANAGE_PERMISSIONS : [ self.get_private_user_role( user, auto_create=True ) ] } - #new_user_dataset_access_role_default_private is set as True in config file - if default_access_private: - permissions[ self.permitted_actions.DATASET_ACCESS ] = permissions.values()[ 0 ] - # Delete all of the current default permissions for the user - for dup in user.default_permissions: - self.sa_session.delete( dup ) - flush_needed = True - # Add the new default permissions for the user - for action, roles in permissions.items(): - if isinstance( action, Action ): - action = action.action - for dup in [ self.model.DefaultUserPermissions( user, action, role ) for role in roles ]: - self.sa_session.add( dup ) - flush_needed = True - if flush_needed: - self.sa_session.flush() - if history: - for history in user.active_histories: - self.history_set_default_permissions( history, permissions=permissions, dataset=dataset, bypass_manage_permission=bypass_manage_permission ) - def user_get_default_permissions( self, user ): - permissions = {} - for dup in user.default_permissions: - action = self.get_action( dup.action ) - if action in permissions: - permissions[ action ].append( dup.role ) - else: - permissions[ action ] = [ dup.role ] - return permissions - def history_set_default_permissions( self, history, permissions={}, dataset=False, bypass_manage_permission=False ): - # bypass_manage_permission is used to change permissions of datasets in a user-less history when logging in - flush_needed = False - user = history.user - if not user: - # default permissions on a user-less history are None - return None - if not permissions: - permissions = self.user_get_default_permissions( user ) - # Delete all of the current default permission for the history - for dhp in history.default_permissions: - self.sa_session.delete( dhp ) - flush_needed = True - # Add the new default permissions for the history - for action, roles in permissions.items(): - if isinstance( action, Action ): - action = action.action - for dhp in [ self.model.DefaultHistoryPermissions( history, action, role ) for role in roles ]: - self.sa_session.add( dhp ) - flush_needed = True - if flush_needed: - self.sa_session.flush() - if dataset: - # Only deal with datasets that are not purged - for hda in history.activatable_datasets: - dataset = hda.dataset - if dataset.library_associations: - # Don't change permissions on a dataset associated with a library - continue - if [ assoc for assoc in dataset.history_associations if assoc.history not in user.histories ]: - # Don't change permissions on a dataset associated with a history not owned by the user - continue - if bypass_manage_permission or self.can_manage_dataset( user.all_roles(), dataset ): - self.set_all_dataset_permissions( dataset, permissions ) - def history_get_default_permissions( self, history ): - permissions = {} - for dhp in history.default_permissions: - action = self.get_action( dhp.action ) - if action in permissions: - permissions[ action ].append( dhp.role ) - else: - permissions[ action ] = [ dhp.role ] - return permissions - def set_all_dataset_permissions( self, dataset, permissions={} ): - """ - Set new permissions on a dataset, eliminating all current permissions - permissions looks like: { Action : [ Role, Role ] } - """ - # Make sure that DATASET_MANAGE_PERMISSIONS is associated with at least 1 role - has_dataset_manage_permissions = False - for action, roles in permissions.items(): - if isinstance( action, Action ): - if action == self.permitted_actions.DATASET_MANAGE_PERMISSIONS and roles: - has_dataset_manage_permissions = True - break - elif action == self.permitted_actions.DATASET_MANAGE_PERMISSIONS.action and roles: - has_dataset_manage_permissions = True - break - if not has_dataset_manage_permissions: - return "At least 1 role must be associated with the manage permissions permission on this dataset." - flush_needed = False - # Delete all of the current permissions on the dataset - for dp in dataset.actions: - self.sa_session.delete( dp ) - flush_needed = True - # Add the new permissions on the dataset - for action, roles in permissions.items(): - if isinstance( action, Action ): - action = action.action - for dp in [ self.model.DatasetPermissions( action, dataset, role ) for role in roles ]: - self.sa_session.add( dp ) - flush_needed = True - if flush_needed: - self.sa_session.flush() - return "" - def set_dataset_permission( self, dataset, permission={} ): - """ - Set a specific permission on a dataset, leaving all other current permissions on the dataset alone - permissions looks like: { Action : [ Role, Role ] } - """ - flush_needed = False - for action, roles in permission.items(): - if isinstance( action, Action ): - action = action.action - # Delete the current specific permission on the dataset if one exists - for dp in dataset.actions: - if dp.action == action: - self.sa_session.delete( dp ) - flush_needed = True - # Add the new specific permission on the dataset - for dp in [ self.model.DatasetPermissions( action, dataset, role ) for role in roles ]: - self.sa_session.add( dp ) - flush_needed = True - if flush_needed: - self.sa_session.flush() - def get_permissions( self, item ): - """ - Return a dictionary containing the actions and associated roles on item - where item is one of Library, LibraryFolder, LibraryDatasetDatasetAssociation, - LibraryDataset, Dataset. The dictionary looks like: { Action : [ Role, Role ] }. - """ - permissions = {} - for item_permission in item.actions: - action = self.get_action( item_permission.action ) - if action in permissions: - permissions[ action ].append( item_permission.role ) - else: - permissions[ action ] = [ item_permission.role ] - return permissions - def get_accessible_request_types( self, trans, user ): - """Return all RequestTypes that the received user has permission to access.""" - accessible_request_types = [] - current_user_role_ids = [ role.id for role in user.all_roles() ] - request_type_access_action = self.permitted_actions.REQUEST_TYPE_ACCESS.action - restricted_request_type_ids = [ rtp.request_type_id for rtp in trans.sa_session.query( trans.model.RequestTypePermissions ) \ - .filter( trans.model.RequestTypePermissions.table.c.action == request_type_access_action ) \ - .distinct() ] - accessible_restricted_request_type_ids = [ rtp.request_type_id for rtp in trans.sa_session.query( trans.model.RequestTypePermissions ) \ - .filter( and_( trans.model.RequestTypePermissions.table.c.action == request_type_access_action, - trans.model.RequestTypePermissions.table.c.role_id.in_( current_user_role_ids ) ) ) ] - # Filter to get libraries accessible by the current user. Get both - # public libraries and restricted libraries accessible by the current user. - for request_type in trans.sa_session.query( trans.model.RequestType ) \ - .filter( and_( trans.model.RequestType.table.c.deleted == False, - ( or_( not_( trans.model.RequestType.table.c.id.in_( restricted_request_type_ids ) ), - trans.model.RequestType.table.c.id.in_( accessible_restricted_request_type_ids ) ) ) ) ) \ - .order_by( trans.app.model.RequestType.name ): - accessible_request_types.append( request_type ) - return accessible_request_types - def copy_dataset_permissions( self, src, dst ): - if not isinstance( src, self.model.Dataset ): - src = src.dataset - if not isinstance( dst, self.model.Dataset ): - dst = dst.dataset - self.set_all_dataset_permissions( dst, self.get_permissions( src ) ) - def privately_share_dataset( self, dataset, users = [] ): - intersect = None - for user in users: - roles = [ ura.role for ura in user.roles if ura.role.type == self.model.Role.types.SHARING ] - if intersect is None: - intersect = roles - else: - new_intersect = [] - for role in roles: - if role in intersect: - new_intersect.append( role ) - intersect = new_intersect - sharing_role = None - if intersect: - for role in intersect: - if not filter( lambda x: x not in users, [ ura.user for ura in role.users ] ): - # only use a role if it contains ONLY the users we're sharing with - sharing_role = role - break - if sharing_role is None: - sharing_role = self.model.Role( name = "Sharing role for: " + ", ".join( [ u.email for u in users ] ), - type = self.model.Role.types.SHARING ) - self.sa_session.add( sharing_role ) - self.sa_session.flush() - for user in users: - self.associate_components( user=user, role=sharing_role ) - self.set_dataset_permission( dataset, { self.permitted_actions.DATASET_ACCESS : [ sharing_role ] } ) - def set_all_library_permissions( self, trans, library_item, permissions={} ): - # Set new permissions on library_item, eliminating all current permissions - flush_needed = False - for role_assoc in library_item.actions: - self.sa_session.delete( role_assoc ) - flush_needed = True - # Add the new permissions on library_item - for item_class, permission_class in self.library_item_assocs: - if isinstance( library_item, item_class ): - for action, roles in permissions.items(): - if isinstance( action, Action ): - action = action.action - for role_assoc in [ permission_class( action, library_item, role ) for role in roles ]: - self.sa_session.add( role_assoc ) - flush_needed = True - if isinstance( library_item, self.model.LibraryDatasetDatasetAssociation ): - # Permission setting related to DATASET_MANAGE_PERMISSIONS was broken for a period of time, - # so it is possible that some Datasets have no roles associated with the DATASET_MANAGE_PERMISSIONS - # permission. In this case, we'll reset this permission to the library_item user's private role. - if not library_item.dataset.has_manage_permissions_roles( trans ): - permission = {} - permissions[ self.permitted_actions.DATASET_MANAGE_PERMISSIONS ] = [ trans.app.security_agent.get_private_user_role( library_item.user ) ] - self.set_dataset_permission( library_item.dataset, permissions ) - if action == self.permitted_actions.LIBRARY_MANAGE.action and roles: - # Handle the special case when we are setting the LIBRARY_MANAGE_PERMISSION on a - # library_dataset_dataset_association since the roles need to be applied to the - # DATASET_MANAGE_PERMISSIONS permission on the associated dataset. - permissions = {} - permissions[ self.permitted_actions.DATASET_MANAGE_PERMISSIONS ] = roles - self.set_dataset_permission( library_item.dataset, permissions ) - if flush_needed: - self.sa_session.flush() - def library_is_public( self, library, contents=False ): - if contents: - # Check all contained folders and datasets to find any that are not public - if not self.folder_is_public( library.root_folder ): - return False - # A library is considered public if there are no "access" actions associated with it. - return self.permitted_actions.LIBRARY_ACCESS.action not in [ a.action for a in library.actions ] - def make_library_public( self, library, contents=False ): - flush_needed = False - if contents: - # Make all contained folders (include deleted folders, but not purged folders), public - self.make_folder_public( library.root_folder ) - # A library is considered public if there are no LIBRARY_ACCESS actions associated with it. - for lp in library.actions: - if lp.action == self.permitted_actions.LIBRARY_ACCESS.action: - self.sa_session.delete( lp ) - flush_needed = True - if flush_needed: - self.sa_session.flush() - def folder_is_public( self, folder ): - for sub_folder in folder.folders: - if not self.folder_is_public( sub_folder ): - return False - for library_dataset in folder.datasets: - ldda = library_dataset.library_dataset_dataset_association - if ldda and ldda.dataset and not self.dataset_is_public( ldda.dataset ): - return False - return True - def make_folder_public( self, folder ): - # Make all of the contents (include deleted contents, but not purged contents) of folder public - for sub_folder in folder.folders: - if not sub_folder.purged: - self.make_folder_public( sub_folder ) - for library_dataset in folder.datasets: - dataset = library_dataset.library_dataset_dataset_association.dataset - if not dataset.purged and not self.dataset_is_public( dataset ): - self.make_dataset_public( dataset ) - - def dataset_is_public( self, dataset ): - # A dataset is considered public if there are no "access" actions associated with it. Any - # other actions ( 'manage permissions', 'edit metadata' ) are irrelevant. - # SM: Accessing dataset.actions will cause a query to be emitted. - return self.permitted_actions.DATASET_ACCESS.action not in [ a.action for a in dataset.actions ] - - def datasets_are_public( self, trans, datasets ): - ''' - Given a transaction object and a list of Datasets, return - a mapping from Dataset ids to whether the Dataset is public - or not. All Dataset ids should be returned in the mapping's keys. - ''' - # We go the other way around from dataset_is_public: we start with - # all datasets being marked as public. If there is an access action - # associated with the dataset, then we mark it as nonpublic: - datasets_public = {} - dataset_ids = [ dataset.id for dataset in datasets ] - for dataset_id in dataset_ids: - datasets_public[ dataset_id ] = True - - # Now get all datasets which have DATASET_ACCESS actions: - access_data_perms = trans.sa_session.query( trans.app.model.DatasetPermissions ) \ - .filter( and_( trans.app.model.DatasetPermissions.dataset_id.in_( dataset_ids ), - trans.app.model.DatasetPermissions.action == self.permitted_actions.DATASET_ACCESS.action ) ) \ - .all() - # Every dataset returned has "access" privileges associated with it, - # so it's not public. - for permission in access_data_perms: - datasets_public[ permission.dataset_id ] = False - return datasets_public - - - def make_dataset_public( self, dataset ): - # A dataset is considered public if there are no "access" actions associated with it. Any - # other actions ( 'manage permissions', 'edit metadata' ) are irrelevant. - flush_needed = False - for dp in dataset.actions: - if dp.action == self.permitted_actions.DATASET_ACCESS.action: - self.sa_session.delete( dp ) - flush_needed = True - if flush_needed: - self.sa_session.flush() - def derive_roles_from_access( self, trans, item_id, cntrller, library=False, **kwd ): - # Check the access permission on a dataset. If library is true, item_id refers to a library. If library - # is False, item_id refers to a dataset ( item_id must currently be decoded before being sent ). The - # cntrller param is the calling controller, which needs to be passed to get_legitimate_roles(). - msg = '' - permissions = {} - # accessible will be True only if at least 1 user has every role in DATASET_ACCESS_in - accessible = False - # legitimate will be True only if all roles in DATASET_ACCESS_in are in the set of roles returned from - # get_legitimate_roles() - legitimate = False - # private_role_found will be true only if more than 1 role is being associated with the DATASET_ACCESS - # permission on item, and at least 1 of the roles is private. - private_role_found = False - error = False - for k, v in get_permitted_actions( filter='DATASET' ).items(): - in_roles = [ self.sa_session.query( self.model.Role ).get( x ) for x in listify( kwd.get( k + '_in', [] ) ) ] - if v == self.permitted_actions.DATASET_ACCESS and in_roles: - if library: - item = self.sa_session.query( self.model.Library ).get( item_id ) - else: - item = self.sa_session.query( self.model.Dataset ).get( item_id ) - if ( library and not self.library_is_public( item ) ) or ( not library and not self.dataset_is_public( item ) ): - # Ensure that roles being associated with DATASET_ACCESS are a subset of the legitimate roles - # derived from the roles associated with the access permission on item if it's not public. This - # will keep ill-legitimate roles from being associated with the DATASET_ACCESS permission on the - # dataset (i.e., in the case where item is .a library, if Role1 is associated with LIBRARY_ACCESS, - # then only those users that have Role1 should be associated with DATASET_ACCESS. - legitimate_roles = self.get_legitimate_roles( trans, item, cntrller ) - ill_legitimate_roles = [] - for role in in_roles: - if role not in legitimate_roles: - ill_legitimate_roles.append( role ) - if ill_legitimate_roles: - # This condition should never occur since ill-legitimate roles are filtered out of the set of - # roles displayed on the forms, but just in case there is a bug somewhere that incorrectly - # filters, we'll display this message. - error = True - msg += "The following roles are not associated with users that have the 'access' permission on this " - msg += "item, so they were incorrectly displayed: " - for role in ill_legitimate_roles: - msg += "%s, " % role.name - msg = msg.rstrip( ", " ) - new_in_roles = [] - for role in in_roles: - if role in legitimate_roles: - new_in_roles.append( role ) - in_roles = new_in_roles - else: - legitimate = True - if len( in_roles ) > 1: - # At least 1 user must have every role associated with the access - # permission on this dataset, or the dataset is not accessible. - # Since we have more than 1 role, none of them can be private. - for role in in_roles: - if role.type == self.model.Role.types.PRIVATE: - private_role_found = True - break - if len( in_roles ) == 1: - accessible = True - else: - # At least 1 user must have every role associated with the access - # permission on this dataset, or the dataset is not accessible. - in_roles_set = set() - for role in in_roles: - in_roles_set.add( role ) - users_set = set() - for role in in_roles: - for ura in role.users: - users_set.add( ura.user ) - for gra in role.groups: - group = gra.group - for uga in group.users: - users_set.add( uga.user ) - # Make sure that at least 1 user has every role being associated with the dataset. - for user in users_set: - user_roles_set = set() - for ura in user.roles: - user_roles_set.add( ura.role ) - if in_roles_set.issubset( user_roles_set ): - accessible = True - break - if private_role_found or not accessible: - error = True - # Don't set the permissions for DATASET_ACCESS if inaccessible or multiple roles with - # at least 1 private, but set all other permissions. - permissions[ self.get_action( v.action ) ] = [] - msg = "At least 1 user must have every role associated with accessing datasets. " - if private_role_found: - msg += "Since you are associating more than 1 role, no private roles are allowed." - if not accessible: - msg += "The roles you attempted to associate for access would make the datasets in-accessible by everyone." - else: - permissions[ self.get_action( v.action ) ] = in_roles - else: - permissions[ self.get_action( v.action ) ] = in_roles - return permissions, in_roles, error, msg - def copy_library_permissions( self, trans, source_library_item, target_library_item, user=None ): - # Copy all relevant permissions from source. - permissions = {} - for role_assoc in source_library_item.actions: - if role_assoc.action != self.permitted_actions.LIBRARY_ACCESS.action: - # LIBRARY_ACCESS is a special permission that is set only at the library level. - if role_assoc.action in permissions: - permissions[role_assoc.action].append( role_assoc.role ) - else: - permissions[role_assoc.action] = [ role_assoc.role ] - self.set_all_library_permissions( trans, target_library_item, permissions ) - if user: - item_class = None - for item_class, permission_class in self.library_item_assocs: - if isinstance( target_library_item, item_class ): - break - if item_class: - # Make sure user's private role is included - private_role = self.model.security_agent.get_private_user_role( user ) - for name, action in self.permitted_actions.items(): - if not permission_class.filter_by( role_id = private_role.id, action = action.action ).first(): - lp = permission_class( action.action, target_library_item, private_role ) - self.sa_session.add( lp ) - self.sa_session.flush() - else: - raise 'Invalid class (%s) specified for target_library_item (%s)' % \ - ( target_library_item.__class__, target_library_item.__class__.__name__ ) - def get_permitted_libraries( self, trans, user, actions ): - """ - This method is historical (it is not currently used), but may be useful again at some - point. It returns a dictionary whose keys are library objects and whose values are a - comma-separated string of folder ids. This method works with the show_library_item() - method below, and it returns libraries for which the received user has permission to - perform the received actions. Here is an example call to this method to return all - libraries for which the received user has LIBRARY_ADD permission:: - - libraries = trans.app.security_agent.get_permitted_libraries( trans, user, - [ trans.app.security_agent.permitted_actions.LIBRARY_ADD ] ) - """ - all_libraries = trans.sa_session.query( trans.app.model.Library ) \ - .filter( trans.app.model.Library.table.c.deleted == False ) \ - .order_by( trans.app.model.Library.name ) - roles = user.all_roles() - actions_to_check = actions - # The libraries dictionary looks like: { library : '1,2' }, library : '3' } - # Its keys are the libraries that should be displayed for the current user and whose values are a - # string of comma-separated folder ids, of the associated folders the should NOT be displayed. - # The folders that should not be displayed may not be a complete list, but it is ultimately passed - # to the calling method to keep from re-checking the same folders when the library / folder - # select lists are rendered. - libraries = {} - for library in all_libraries: - can_show, hidden_folder_ids = self.show_library_item( self, roles, library, actions_to_check ) - if can_show: - libraries[ library ] = hidden_folder_ids - return libraries - def show_library_item( self, user, roles, library_item, actions_to_check, hidden_folder_ids='' ): - """ - This method must be sent an instance of Library() or LibraryFolder(). Recursive execution produces a - comma-separated string of folder ids whose folders do NOT meet the criteria for showing. Along with - the string, True is returned if the current user has permission to perform any 1 of actions_to_check - on library_item. Otherwise, cycle through all sub-folders in library_item until one is found that meets - this criteria, if it exists. This method does not necessarily scan the entire library as it returns - when it finds the first library_item that allows user to perform any one action in actions_to_check. - """ - for action in actions_to_check: - if self.allow_action( roles, action, library_item ): - return True, hidden_folder_ids - if isinstance( library_item, self.model.Library ): - return self.show_library_item( user, roles, library_item.root_folder, actions_to_check, hidden_folder_ids='' ) - if isinstance( library_item, self.model.LibraryFolder ): - for folder in library_item.active_folders: - can_show, hidden_folder_ids = self.show_library_item( user, roles, folder, actions_to_check, hidden_folder_ids=hidden_folder_ids ) - if can_show: - return True, hidden_folder_ids - if hidden_folder_ids: - hidden_folder_ids = '%s,%d' % ( hidden_folder_ids, folder.id ) - else: - hidden_folder_ids = '%d' % folder.id - return False, hidden_folder_ids - def get_showable_folders( self, user, roles, library_item, actions_to_check, hidden_folder_ids=[], showable_folders=[] ): - """ - This method must be sent an instance of Library(), all the folders of which are scanned to determine if - user is allowed to perform any action in actions_to_check. The param hidden_folder_ids, if passed, should - contain a list of folder IDs which was generated when the library was previously scanned - using the same actions_to_check. A list of showable folders is generated. This method scans the entire library. - """ - if isinstance( library_item, self.model.Library ): - return self.get_showable_folders( user, roles, library_item.root_folder, actions_to_check, showable_folders=[] ) - if isinstance( library_item, self.model.LibraryFolder ): - if library_item.id not in hidden_folder_ids: - for action in actions_to_check: - if self.allow_action( roles, action, library_item ): - showable_folders.append( library_item ) - break - for folder in library_item.active_folders: - self.get_showable_folders( user, roles, folder, actions_to_check, showable_folders=showable_folders ) - return showable_folders - def set_entity_user_associations( self, users=[], roles=[], groups=[], delete_existing_assocs=True ): - for user in users: - if delete_existing_assocs: - flush_needed = False - for a in user.non_private_roles + user.groups: - self.sa_session.delete( a ) - flush_needed = True - if flush_needed: - self.sa_session.flush() - self.sa_session.refresh( user ) - for role in roles: - # Make sure we are not creating an additional association with a PRIVATE role - if role not in user.roles: - self.associate_components( user=user, role=role ) - for group in groups: - self.associate_components( user=user, group=group ) - def set_entity_group_associations( self, groups=[], users=[], roles=[], delete_existing_assocs=True ): - for group in groups: - if delete_existing_assocs: - flush_needed = False - for a in group.roles + group.users: - self.sa_session.delete( a ) - flush_needed = True - if flush_needed: - self.sa_session.flush() - for role in roles: - self.associate_components( group=group, role=role ) - for user in users: - self.associate_components( group=group, user=user ) - def set_entity_role_associations( self, roles=[], users=[], groups=[], delete_existing_assocs=True ): - for role in roles: - if delete_existing_assocs: - flush_needed = False - for a in role.users + role.groups: - self.sa_session.delete( a ) - flush_needed = True - if flush_needed: - self.sa_session.flush() - for user in users: - self.associate_components( user=user, role=role ) - for group in groups: - self.associate_components( group=group, role=role ) - def get_component_associations( self, **kwd ): - assert len( kwd ) == 2, 'You must specify exactly 2 Galaxy security components to check for associations.' - if 'dataset' in kwd: - if 'action' in kwd: - return self.sa_session.query( self.model.DatasetPermissions ).filter_by( action = kwd['action'].action, dataset_id = kwd['dataset'].id ).first() - elif 'user' in kwd: - if 'group' in kwd: - return self.sa_session.query( self.model.UserGroupAssociation ).filter_by( group_id = kwd['group'].id, user_id = kwd['user'].id ).first() - elif 'role' in kwd: - return self.sa_session.query( self.model.UserRoleAssociation ).filter_by( role_id = kwd['role'].id, user_id = kwd['user'].id ).first() - elif 'group' in kwd: - if 'role' in kwd: - return self.sa_session.query( self.model.GroupRoleAssociation ).filter_by( role_id = kwd['role'].id, group_id = kwd['group'].id ).first() - raise 'No valid method of associating provided components: %s' % kwd - def check_folder_contents( self, user, roles, folder, hidden_folder_ids='' ): - """ - This method must always be sent an instance of LibraryFolder(). Recursive execution produces a - comma-separated string of folder ids whose folders do NOT meet the criteria for showing. Along - with the string, True is returned if the current user has permission to access folder. Otherwise, - cycle through all sub-folders in folder until one is found that meets this criteria, if it exists. - This method does not necessarily scan the entire library as it returns when it finds the first - folder that is accessible to user. - """ - # If a folder is writeable, it's accessable and we need not go further - if self.can_add_library_item( roles, folder ): - return True, '' - action = self.permitted_actions.DATASET_ACCESS - - lddas = self.sa_session.query( self.model.LibraryDatasetDatasetAssociation ) \ - .join( "library_dataset" ) \ - .filter( self.model.LibraryDataset.folder == folder ) \ - .join( "dataset" ) \ - .options( eagerload_all( "dataset.actions" ) ) \ - .all() - - for ldda in lddas: - ldda_access_permissions = self.get_item_actions( action, ldda.dataset ) - if not ldda_access_permissions: - # Dataset is public - return True, hidden_folder_ids - for ldda_access_permission in ldda_access_permissions: - if ldda_access_permission.role in roles: - # The current user has access permission on the dataset - return True, hidden_folder_ids - for sub_folder in folder.active_folders: - can_access, hidden_folder_ids = self.check_folder_contents( user, roles, sub_folder, hidden_folder_ids=hidden_folder_ids ) - if can_access: - return True, hidden_folder_ids - if hidden_folder_ids: - hidden_folder_ids = '%s,%d' % ( hidden_folder_ids, sub_folder.id ) - else: - hidden_folder_ids = '%d' % sub_folder.id - return False, hidden_folder_ids - def can_access_request_type( self, roles, request_type ): - action = self.permitted_actions.REQUEST_TYPE_ACCESS - request_type_actions = [] - for permission in request_type.actions: - if permission.action == action.action: - request_type_actions.append( permission ) - if not request_type_actions: - return True - ret_val = False - for request_type_action in request_type_actions: - if request_type_action.role in roles: - ret_val = True - break - return ret_val - def set_request_type_permissions( self, request_type, permissions={} ): - # Set new permissions on request_type, eliminating all current permissions - for role_assoc in request_type.actions: - self.sa_session.delete( role_assoc ) - # Add the new permissions on request_type - item_class = self.model.RequestType - permission_class = self.model.RequestTypePermissions - flush_needed = False - for action, roles in permissions.items(): - if isinstance( action, Action ): - action = action.action - for role_assoc in [ permission_class( action, request_type, role ) for role in roles ]: - self.sa_session.add( role_assoc ) - flush_needed = True - if flush_needed: - self.sa_session.flush() - -class HostAgent( RBACAgent ): - """ - A simple security agent which allows access to datasets based on host. - This exists so that externals sites such as UCSC can gain access to - datasets which have permissions which would normally prevent such access. - """ - # TODO: Make sites user configurable - sites = Bunch( - ucsc_main = ( 'hgw1.cse.ucsc.edu', 'hgw2.cse.ucsc.edu', 'hgw3.cse.ucsc.edu', 'hgw4.cse.ucsc.edu', - 'hgw5.cse.ucsc.edu', 'hgw6.cse.ucsc.edu', 'hgw7.cse.ucsc.edu', 'hgw8.cse.ucsc.edu' ), - ucsc_test = ( 'hgwdev.cse.ucsc.edu', ), - ucsc_archaea = ( 'lowepub.cse.ucsc.edu', ) - ) - def __init__( self, model, permitted_actions=None ): - self.model = model - if permitted_actions: - self.permitted_actions = permitted_actions - @property - def sa_session( self ): - """Returns a SQLAlchemy session""" - return self.model.context - def allow_action( self, addr, action, **kwd ): - if 'dataset' in kwd and action == self.permitted_actions.DATASET_ACCESS: - hda = kwd['dataset'] - if action == self.permitted_actions.DATASET_ACCESS and action.action not in [ dp.action for dp in hda.dataset.actions ]: - log.debug( 'Allowing access to public dataset with hda: %i.' % hda.id ) - return True # dataset has no roles associated with the access permission, thus is already public - hdadaa = self.sa_session.query( self.model.HistoryDatasetAssociationDisplayAtAuthorization ) \ - .filter_by( history_dataset_association_id = hda.id ).first() - if not hdadaa: - log.debug( 'Denying access to private dataset with hda: %i. No hdadaa record for this dataset.' % hda.id ) - return False # no auth - # We could just look up the reverse of addr, but then we'd also - # have to verify it with the forward address and special case any - # IPs (instead of hosts) in the server list. - # - # This would be improved by caching, but that's what the OS's name - # service cache daemon is for (you ARE running nscd, right?). - for server in HostAgent.sites.get( hdadaa.site, [] ): - # We're going to search in order, but if the remote site is load - # balancing their connections (as UCSC does), this is okay. - try: - if socket.gethostbyname( server ) == addr: - break # remote host is in the server list - except ( socket.error, socket.gaierror ): - pass # can't resolve, try next - else: - log.debug( 'Denying access to private dataset with hda: %i. Remote addr is not a valid server for site: %s.' % ( hda.id, hdadaa.site ) ) - return False # remote addr is not in the server list - if ( datetime.utcnow() - hdadaa.update_time ) > timedelta( seconds=60 ): - log.debug( 'Denying access to private dataset with hda: %i. Authorization was granted, but has expired.' % hda.id ) - return False # not authz'd in the last 60 seconds - log.debug( 'Allowing access to private dataset with hda: %i. Remote server is: %s.' % ( hda.id, server ) ) - return True - else: - raise 'The dataset access permission is the only valid permission in the host security agent.' - def set_dataset_permissions( self, hda, user, site ): - hdadaa = self.sa_session.query( self.model.HistoryDatasetAssociationDisplayAtAuthorization ) \ - .filter_by( history_dataset_association_id = hda.id ).first() - if hdadaa: - hdadaa.update_time = datetime.utcnow() - else: - hdadaa = self.model.HistoryDatasetAssociationDisplayAtAuthorization( hda=hda, user=user, site=site ) - self.sa_session.add( hdadaa ) - self.sa_session.flush() - -def get_permitted_actions( filter=None ): - '''Utility method to return a subset of RBACAgent's permitted actions''' - if filter is None: - return RBACAgent.permitted_actions - tmp_bunch = Bunch() - [ tmp_bunch.__dict__.__setitem__(k, v) for k, v in RBACAgent.permitted_actions.items() if k.startswith( filter ) ] - return tmp_bunch +""" +Galaxy Security + +""" +import logging, socket, operator +from datetime import datetime, timedelta +from galaxy.util.bunch import Bunch +from galaxy.util import listify +from galaxy.model.orm import * + +log = logging.getLogger(__name__) + +class Action( object ): + def __init__( self, action, description, model ): + self.action = action + self.description = description + self.model = model + +class RBACAgent: + """Class that handles galaxy security""" + permitted_actions = Bunch( + DATASET_MANAGE_PERMISSIONS = Action( "manage permissions", "Users having associated role can manage the roles associated with permissions on this dataset", "grant" ), + DATASET_ACCESS = Action( "access", "Users having associated role can import this dataset into their history for analysis", "restrict" ), + LIBRARY_ACCESS = Action( "access library", "Restrict access to this library to only users having associated role", "restrict" ), + LIBRARY_ADD = Action( "add library item", "Users having associated role can add library items to this library item", "grant" ), + LIBRARY_MODIFY = Action( "modify library item", "Users having associated role can modify this library item", "grant" ), + LIBRARY_MANAGE = Action( "manage library permissions", "Users having associated role can manage roles associated with permissions on this library item", "grant" ), + # Request type permissions + REQUEST_TYPE_ACCESS = Action( "access request_type", "Restrict access to this request type to only users having associated role", "restrict" ) + + ) + def get_action( self, name, default=None ): + """Get a permitted action by its dict key or action name""" + for k, v in self.permitted_actions.items(): + if k == name or v.action == name: + return v + return default + def get_actions( self ): + """Get all permitted actions as a list of Action objects""" + return self.permitted_actions.__dict__.values() + def get_item_actions( self, action, item ): + raise 'No valid method of retrieving action (%s) for item %s.' % ( action, item ) + def guess_derived_permissions_for_datasets( self, datasets = [] ): + raise "Unimplemented Method" + def can_access_dataset( self, roles, dataset ): + raise "Unimplemented Method" + def can_manage_dataset( self, roles, dataset ): + raise "Unimplemented Method" + def can_access_library( self, roles, library ): + raise "Unimplemented Method" + def can_add_library_item( self, roles, item ): + raise "Unimplemented Method" + def can_modify_library_item( self, roles, item ): + raise "Unimplemented Method" + def can_manage_library_item( self, roles, item ): + raise "Unimplemented Method" + def associate_components( self, **kwd ): + raise 'No valid method of associating provided components: %s' % kwd + def create_private_user_role( self, user ): + raise "Unimplemented Method" + def get_private_user_role( self, user ): + raise "Unimplemented Method" + def get_accessible_request_types( self, trans, user ): + raise "Unimplemented Method" + def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False ): + raise "Unimplemented Method" + def history_set_default_permissions( self, history, permissions=None, dataset=False, bypass_manage_permission=False ): + raise "Unimplemented Method" + def set_all_dataset_permissions( self, dataset, permissions ): + raise "Unimplemented Method" + def set_dataset_permission( self, dataset, permission ): + raise "Unimplemented Method" + def set_all_library_permissions( self, trans, dataset, permissions ): + raise "Unimplemented Method" + def library_is_public( self, library ): + raise "Unimplemented Method" + def make_library_public( self, library ): + raise "Unimplemented Method" + def get_accessible_libraries( self, trans, user ): + raise "Unimplemented Method" + def get_permitted_libraries( self, trans, user, actions ): + raise "Unimplemented Method" + def folder_is_public( self, library ): + raise "Unimplemented Method" + def make_folder_public( self, folder, count=0 ): + raise "Unimplemented Method" + def dataset_is_public( self, dataset ): + raise "Unimplemented Method" + def make_dataset_public( self, dataset ): + raise "Unimplemented Method" + def get_permissions( self, library_dataset ): + raise "Unimplemented Method" + def get_all_roles( self, trans, cntrller ): + raise "Unimplemented Method" + def get_legitimate_roles( self, trans, item, cntrller ): + raise "Unimplemented Method" + def derive_roles_from_access( self, trans, item_id, cntrller, library=False, **kwd ): + raise "Unimplemented Method" + def get_component_associations( self, **kwd ): + raise "Unimplemented Method" + def components_are_associated( self, **kwd ): + return bool( self.get_component_associations( **kwd ) ) + def convert_permitted_action_strings( self, permitted_action_strings ): + """ + When getting permitted actions from an untrusted source like a + form, ensure that they match our actual permitted actions. + """ + return filter( lambda x: x is not None, [ self.permitted_actions.get( action_string ) for action_string in permitted_action_strings ] ) + +class GalaxyRBACAgent( RBACAgent ): + def __init__( self, model, permitted_actions=None ): + self.model = model + if permitted_actions: + self.permitted_actions = permitted_actions + # List of "library_item" objects and their associated permissions and info template objects + self.library_item_assocs = ( + ( self.model.Library, self.model.LibraryPermissions ), + ( self.model.LibraryFolder, self.model.LibraryFolderPermissions ), + ( self.model.LibraryDataset, self.model.LibraryDatasetPermissions ), + ( self.model.LibraryDatasetDatasetAssociation, self.model.LibraryDatasetDatasetAssociationPermissions ) ) + @property + def sa_session( self ): + """Returns a SQLAlchemy session""" + return self.model.context + def sort_by_attr( self, seq, attr ): + """ + Sort the sequence of objects by object's attribute + Arguments: + seq - the list or any sequence (including immutable one) of objects to sort. + attr - the name of attribute to sort by + """ + # Use the "Schwartzian transform" + # Create the auxiliary list of tuples where every i-th tuple has form + # (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not + # only to provide stable sorting, but mainly to eliminate comparison of objects + # (which can be expensive or prohibited) in case of equal attribute values. + intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq ) + intermed.sort() + return map( operator.getitem, intermed, ( -1, ) * len( intermed ) ) + def get_all_roles( self, trans, cntrller ): + admin_controller = cntrller in [ 'library_admin' ] + roles = set() + if not trans.user: + return trans.sa_session.query( trans.app.model.Role ) \ + .filter( and_( self.model.Role.table.c.deleted==False, + self.model.Role.table.c.type != self.model.Role.types.PRIVATE, + self.model.Role.table.c.type != self.model.Role.types.SHARING ) ) \ + .order_by( self.model.Role.table.c.name ) + if admin_controller: + # The library is public and the user is an admin, so all roles are legitimate + for role in trans.sa_session.query( trans.app.model.Role ) \ + .filter( self.model.Role.table.c.deleted==False ) \ + .order_by( self.model.Role.table.c.name ): + roles.add( role ) + else: + # Add the current user's private role + roles.add( self.get_private_user_role( trans.user ) ) + # Add the current user's sharing roles + for role in self.get_sharing_roles( trans.user ): + roles.add( role ) + # Add all remaining non-private, non-sharing roles + for role in trans.sa_session.query( trans.app.model.Role ) \ + .filter( and_( self.model.Role.table.c.deleted==False, + self.model.Role.table.c.type != self.model.Role.types.PRIVATE, + self.model.Role.table.c.type != self.model.Role.types.SHARING ) ) \ + .order_by( self.model.Role.table.c.name ): + roles.add( role ) + return self.sort_by_attr( [ role for role in roles ], 'name' ) + def get_legitimate_roles( self, trans, item, cntrller ): + """ + Return a sorted list of legitimate roles that can be associated with a permission on + item where item is a Library or a Dataset. The cntrller param is the controller from + which the request is sent. We cannot use trans.user_is_admin() because the controller is + what is important since admin users do not necessarily have permission to do things + on items outside of the admin view. + + If cntrller is from the admin side ( e.g., library_admin ): + + - if item is public, all roles, including private roles, are legitimate. + - if item is restricted, legitimate roles are derived from the users and groups associated + with each role that is associated with the access permission ( i.e., DATASET_MANAGE_PERMISSIONS or + LIBRARY_MANAGE ) on item. Legitimate roles will include private roles. + + If cntrller is not from the admin side ( e.g., root, library ): + + - if item is public, all non-private roles, except for the current user's private role, + are legitimate. + - if item is restricted, legitimate roles are derived from the users and groups associated + with each role that is associated with the access permission on item. Private roles, except + for the current user's private role, will be excluded. + """ + admin_controller = cntrller in [ 'library_admin' ] + roles = set() + if ( isinstance( item, self.model.Library ) and self.library_is_public( item ) ) or \ + ( isinstance( item, self.model.Dataset ) and self.dataset_is_public( item ) ): + return self.get_all_roles( trans, cntrller ) + # If item has roles associated with the access permission, we need to start with them. + access_roles = item.get_access_roles( trans ) + for role in access_roles: + if admin_controller or self.ok_to_display( trans.user, role ): + roles.add( role ) + # Each role potentially has users. We need to find all roles that each of those users have. + for ura in role.users: + user = ura.user + for ura2 in user.roles: + if admin_controller or self.ok_to_display( trans.user, ura2.role ): + roles.add( ura2.role ) + # Each role also potentially has groups which, in turn, have members ( users ). We need to + # find all roles that each group's members have. + for gra in role.groups: + group = gra.group + for uga in group.users: + user = uga.user + for ura in user.roles: + if admin_controller or self.ok_to_display( trans.user, ura.role ): + roles.add( ura.role ) + return self.sort_by_attr( [ role for role in roles ], 'name' ) + def ok_to_display( self, user, role ): + """ + Method for checking if: + - a role is private and is the current user's private role + - a role is a sharing role and belongs to the current user + """ + if user: + if role.type == self.model.Role.types.PRIVATE: + return role == self.get_private_user_role( user ) + if role.type == self.model.Role.types.SHARING: + return role in self.get_sharing_roles( user ) + # If role.type is neither private nor sharing, it's ok to display + return True + return role.type != self.model.Role.types.PRIVATE and role.type != self.model.Role.types.SHARING + + def allow_action( self, roles, action, item ): + """ + Method for checking a permission for the current user ( based on roles ) to perform a + specific action on an item, which must be one of: + Dataset, Library, LibraryFolder, LibraryDataset, LibraryDatasetDatasetAssociation + """ + # SM: Note that calling get_item_actions will emit a query. + item_actions = self.get_item_actions( action, item ) + + if not item_actions: + return action.model == 'restrict' + ret_val = False + # For DATASET_ACCESS only, user must have ALL associated roles + if action == self.permitted_actions.DATASET_ACCESS: + for item_action in item_actions: + if item_action.role not in roles: + break + else: + ret_val = True + # For remaining actions, user must have any associated role + else: + for item_action in item_actions: + if item_action.role in roles: + ret_val = True + break + return ret_val + + + def get_actions_for_items( self, trans, action, permission_items ): + # TODO: Rename this; it's a replacement for get_item_actions, but it + # doesn't represent what it's really doing, which is confusing. + # TODO: Make this work for other classes besides lib_datasets. + # That should be as easy as checking the type and writing a query for each; + # we're avoiding using the SQLAlchemy backrefs because they can cause lots + # of queries to be generated. + # + # Originally, get_item_actions did: + # return [ permission for permission in item.actions if permission.action == action.action ] + # The "item" can be just about anything with permissions, and referencing + # item.actions causes the item's permissions to be retrieved. + # This method will retrieve all permissions for all "items" and only + # return the permissions associated with that given action. + # We initialize the permissions list to be empty; we will return an + # empty list by default. + # + # If the dataset id has no corresponding action in its permissions, + # then the returned permissions will not carry an entry for the dataset. + ret_permissions = {} + if ( len( permission_items ) > 0 ): + # SM: NB: LibraryDatasets became Datasets for some odd reason. + if ( isinstance( permission_items[0], trans.model.LibraryDataset ) ): + ids = [ item.library_dataset_id for item in permission_items ] + permissions = trans.sa_session.query( trans.model.LibraryDatasetPermissions ) \ + .filter( and_( trans.model.LibraryDatasetPermissions.library_dataset_id.in_( ids ), + trans.model.LibraryDatasetPermissions.action == action.action ) ) \ + .all() + + # Massage the return data. We will return a list of permissions + # for each library dataset. So we initialize the return list to + # have an empty list for each dataset. Then each permission is + # appended to the right lib dataset. + # TODO: Consider eliminating the initialization and just return + # empty values for each library dataset id. + for item in permission_items: + ret_permissions[ item.library_dataset_id ] = [] + for permission in permissions: + ret_permissions[ permission.library_dataset_id ].append( permission ) + elif ( isinstance( permission_items[0], trans.model.Dataset ) ): + ids = [ item.id for item in permission_items ] + permissions = trans.sa_session.query( trans.model.DatasetPermissions ) \ + .filter( and_( trans.model.DatasetPermissions.dataset_id.in_( ids ), + trans.model.DatasetPermissions.action == action.action ) ) \ + .all() + + # Massage the return data. We will return a list of permissions + # for each library dataset. So we initialize the return list to + # have an empty list for each dataset. Then each permission is + # appended to the right lib dataset. + # TODO: Consider eliminating the initialization and just return + # empty values for each library dataset id. + for item in permission_items: + ret_permissions[ item.id ] = [] + for permission in permissions: + ret_permissions[ permission.dataset_id ].append( permission ) + + # Test that we get the same response from get_item_actions each item: + test_code = False + if test_code: + try: + log.debug( "get_actions_for_items: Test start" ) + for item in permission_items: + base_result = self.get_item_actions( action, item ) + new_result = ret_permissions[ item.library_dataset_id ] + # For now, just test against LibraryDatasetIds; other classes + # are not tested yet. + if len( base_result ) == len( new_result ): + common_result = set(base_result).intersection( new_result ) + if len( common_result ) == len( base_result ): + log.debug( "Match on permissions for id %d" % + item.library_dataset_id ) + # TODO: Fix this failure message: + else: + log.debug( "Error: dataset %d; originally: %s; now: %s" + % ( item.library_dataset_id, + base_result, new_result ) ) + else: + log.debug( "Error: dataset %d: had %d entries, now %d entries" + % ( item.library_dataset_id, len( base_result ), + len( new_result ) ) ) + log.debug( "get_actions_for_items: Test end" ) + except Exception, e: + log.debug( "Exception in test code: %s" % e ) + + return ret_permissions + + + def allow_action_on_libitems( self, trans, user_roles, action, items ): + """ + This should be the equivalent of allow_action defined on multiple items. + It is meant to specifically replace allow_action for multiple + LibraryDatasets, but it could be reproduced or modified for + allow_action's permitted classes - Dataset, Library, LibraryFolder, and + LDDAs. + """ + all_items_actions = self.get_actions_for_items( trans, action, items ) + ret_allow_action = {} + + # Change item to lib_dataset or vice-versa. + for item in items: + if all_items_actions.has_key( item.id ): + item_actions = all_items_actions[ item.id ] + + if self.permitted_actions.DATASET_ACCESS == action: + ret_allow_action[ item.id ] = True + for item_action in item_actions: + if item_action.role not in user_roles: + ret_allow_action[ item.id ] = False + break + + # Else look for just one dataset role to be in the list of + # acceptable user roles: + else: + ret_allow_action[ item.id ] = False + for item_action in item_actions: + if item_action.role in user_roles: + ret_allow_action[ item.id ] = True + break + + else: + if 'restrict' == action.model: + ret_allow_action[ item.id ] = True + else: + ret_allow_action[ item.id ] = False + + # Test it: the result for each dataset should match the result for + # allow_action: + test_code = False + if test_code: + log.debug( "allow_action_for_items: test start" ) + for item in items: + orig_value = self.allow_action( user_roles, action, item ) + if orig_value == ret_allow_action[ item.id ]: + log.debug( "Item %d: success" % item.id ) + else: + log.debug( "Item %d: fail: original: %s; new: %s" + % ( item.id, orig_value, ret_allow_action[ item.id ] ) ) + log.debug( "allow_action_for_items: test end" ) + return ret_allow_action + + + # DELETEME: SM: DO NOT TOUCH! This actually works. + def dataset_access_mapping( self, trans, user_roles, datasets ): + ''' + For the given list of datasets, return a mapping of the datasets' ids + to whether they can be accessed by the user or not. The datasets input + is expected to be a simple list of Dataset objects. + ''' + datasets_public_map = self.datasets_are_public( trans, datasets ) + datasets_allow_action_map = self.allow_action_on_libitems( trans, user_roles, self.permitted_actions.DATASET_ACCESS, datasets ) + can_access = {} + for dataset in datasets: + can_access[ dataset.id ] = datasets_public_map[ dataset.id ] or datasets_allow_action_map[ dataset.id ] + return can_access + + def dataset_permission_map_for_access( self, trans, user_roles, libitems ): + ''' + For a given list of library items (e.g., Datasets), return a map of the + datasets' ids to whether they can have permission to use that action + (e.g., "access" or "modify") on the dataset. The libitems input is + expected to be a simple list of library items, such as Datasets or + LibraryDatasets. + NB: This is currently only usable for Datasets; it was intended to + be used for any library item. + ''' + # Map the library items to whether they are publicly accessible or not. + # Then determine what actions are allowed on the item (in case it's not + # public). Finally, the item is accessible if it's publicly available + # or the right permissions are enabled. + # TODO: This only works for Datasets; other code is using X_is_public, + # so this will have to be rewritten to support other items. + libitems_public_map = self.datasets_are_public( trans, libitems ) + libitems_allow_action_map = self.allow_action_on_libitems( + trans, user_roles, self.permitted_actions.DATASET_ACCESS, libitems ) + can_access = {} + for libitem in libitems: + can_access[ libitem.id ] = libitems_public_map[ libitem.id ] or libitems_allow_action_map[ libitem.id ] + return can_access + + def item_permission_map_for_modify( self, trans, user_roles, libitems ): + return self.allow_action_on_libitems( + trans, user_roles, self.permitted_actions.LIBRARY_MODIFY, libitems ) + + def item_permission_map_for_manage( self, trans, user_roles, libitems ): + return self.allow_action_on_libitems( + trans, user_roles, self.permitted_actions.LIBRARY_MANAGE, libitems ) + + def item_permission_map_for_add( self, trans, user_roles, libitems ): + return self.allow_action_on_libitems( + trans, user_roles, self.permitted_actions.LIBRARY_ADD, libitems ) + + def can_access_dataset( self, user_roles, dataset ): + # SM: dataset_is_public will access dataset.actions, which is a + # backref that causes a query to be made to DatasetPermissions + retval = self.dataset_is_public( dataset ) or self.allow_action( user_roles, self.permitted_actions.DATASET_ACCESS, dataset ) + return retval + + def can_manage_dataset( self, roles, dataset ): + return self.allow_action( roles, self.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset ) + def can_access_library( self, roles, library ): + return self.library_is_public( library ) or self.allow_action( roles, self.permitted_actions.LIBRARY_ACCESS, library ) + def get_accessible_libraries( self, trans, user ): + """Return all data libraries that the received user can access""" + accessible_libraries = [] + current_user_role_ids = [ role.id for role in user.all_roles() ] + library_access_action = self.permitted_actions.LIBRARY_ACCESS.action + restricted_library_ids = [ lp.library_id for lp in trans.sa_session.query( trans.model.LibraryPermissions ) \ + .filter( trans.model.LibraryPermissions.table.c.action == library_access_action ) \ + .distinct() ] + accessible_restricted_library_ids = [ lp.library_id for lp in trans.sa_session.query( trans.model.LibraryPermissions ) \ + .filter( and_( trans.model.LibraryPermissions.table.c.action == library_access_action, + trans.model.LibraryPermissions.table.c.role_id.in_( current_user_role_ids ) ) ) ] + # Filter to get libraries accessible by the current user. Get both + # public libraries and restricted libraries accessible by the current user. + for library in trans.sa_session.query( trans.model.Library ) \ + .filter( and_( trans.model.Library.table.c.deleted == False, + ( or_( not_( trans.model.Library.table.c.id.in_( restricted_library_ids ) ), + trans.model.Library.table.c.id.in_( accessible_restricted_library_ids ) ) ) ) ) \ + .order_by( trans.app.model.Library.name ): + accessible_libraries.append( library ) + return accessible_libraries + def has_accessible_folders( self, trans, folder, user, roles, search_downward=True ): + if self.has_accessible_library_datasets( trans, folder, user, roles, search_downward=search_downward ) or \ + self.can_add_library_item( roles, folder ) or \ + self.can_modify_library_item( roles, folder ) or \ + self.can_manage_library_item( roles, folder ): + return True + if search_downward: + for folder in folder.active_folders: + return self.has_accessible_folders( trans, folder, user, roles, search_downward=search_downward ) + return False + def has_accessible_library_datasets( self, trans, folder, user, roles, search_downward=True ): + for library_dataset in trans.sa_session.query( trans.model.LibraryDataset ) \ + .filter( and_( trans.model.LibraryDataset.table.c.deleted == False, + trans.app.model.LibraryDataset.table.c.folder_id==folder.id ) ): + if self.can_access_library_item( roles, library_dataset, user ): + return True + if search_downward: + return self.__active_folders_have_accessible_library_datasets( trans, folder, user, roles ) + return False + def __active_folders_have_accessible_library_datasets( self, trans, folder, user, roles ): + for active_folder in folder.active_folders: + if self.has_accessible_library_datasets( trans, active_folder, user, roles ): + return True + return False + def can_access_library_item( self, roles, item, user ): + if type( item ) == self.model.Library: + return self.can_access_library( roles, item ) + elif type( item ) == self.model.LibraryFolder: + return self.can_access_library( roles, item.parent_library ) and self.check_folder_contents( user, roles, item )[0] + elif type( item ) == self.model.LibraryDataset: + return self.can_access_library( roles, item.folder.parent_library ) and self.can_access_dataset( roles, item.library_dataset_dataset_association.dataset ) + elif type( item ) == self.model.LibraryDatasetDatasetAssociation: + return self.can_access_library( roles, item.library_dataset.folder.parent_library ) and self.can_access_dataset( roles, item.dataset ) + else: + log.warning( 'Unknown library item type: %s' % type ( item ) ) + return False + def can_add_library_item( self, roles, item ): + return self.allow_action( roles, self.permitted_actions.LIBRARY_ADD, item ) + def can_modify_library_item( self, roles, item ): + return self.allow_action( roles, self.permitted_actions.LIBRARY_MODIFY, item ) + def can_manage_library_item( self, roles, item ): + return self.allow_action( roles, self.permitted_actions.LIBRARY_MANAGE, item ) + + def get_item_actions( self, action, item ): + # item must be one of: Dataset, Library, LibraryFolder, LibraryDataset, LibraryDatasetDatasetAssociation + # SM: Accessing item.actions emits a query to Library_Dataset_Permissions + # if the item is a LibraryDataset: + # TODO: Pass in the item's actions - the item isn't needed + return [ permission for permission in item.actions if permission.action == action.action ] + + def guess_derived_permissions_for_datasets( self, datasets=[] ): + """Returns a dict of { action : [ role, role, ... ] } for the output dataset based upon provided datasets""" + perms = {} + for dataset in datasets: + if not isinstance( dataset, self.model.Dataset ): + dataset = dataset.dataset + these_perms = {} + # initialize blank perms + for action in self.get_actions(): + these_perms[ action ] = [] + # collect this dataset's perms + these_perms = self.get_permissions( dataset ) + # join or intersect this dataset's permissions with others + for action, roles in these_perms.items(): + if action not in perms.keys(): + perms[ action ] = roles + else: + if action.model == 'grant': + # intersect existing roles with new roles + perms[ action ] = filter( lambda x: x in perms[ action ], roles ) + elif action.model == 'restrict': + # join existing roles with new roles + perms[ action ].extend( filter( lambda x: x not in perms[ action ], roles ) ) + return perms + def associate_components( self, **kwd ): + if 'user' in kwd: + if 'group' in kwd: + return self.associate_user_group( kwd['user'], kwd['group'] ) + elif 'role' in kwd: + return self.associate_user_role( kwd['user'], kwd['role'] ) + elif 'role' in kwd: + if 'group' in kwd: + return self.associate_group_role( kwd['group'], kwd['role'] ) + if 'action' in kwd: + if 'dataset' in kwd and 'role' in kwd: + return self.associate_action_dataset_role( kwd['action'], kwd['dataset'], kwd['role'] ) + raise 'No valid method of associating provided components: %s' % kwd + def associate_user_group( self, user, group ): + assoc = self.model.UserGroupAssociation( user, group ) + self.sa_session.add( assoc ) + self.sa_session.flush() + return assoc + def associate_user_role( self, user, role ): + assoc = self.model.UserRoleAssociation( user, role ) + self.sa_session.add( assoc ) + self.sa_session.flush() + return assoc + def associate_group_role( self, group, role ): + assoc = self.model.GroupRoleAssociation( group, role ) + self.sa_session.add( assoc ) + self.sa_session.flush() + return assoc + def associate_action_dataset_role( self, action, dataset, role ): + assoc = self.model.DatasetPermissions( action, dataset, role ) + self.sa_session.add( assoc ) + self.sa_session.flush() + return assoc + def create_private_user_role( self, user ): + # Create private role + role = self.model.Role( name=user.email, description='Private Role for ' + user.email, type=self.model.Role.types.PRIVATE ) + self.sa_session.add( role ) + self.sa_session.flush() + # Add user to role + self.associate_components( role=role, user=user ) + return role + def get_private_user_role( self, user, auto_create=False ): + role = self.sa_session.query( self.model.Role ) \ + .filter( and_( self.model.Role.table.c.name == user.email, + self.model.Role.table.c.type == self.model.Role.types.PRIVATE ) ) \ + .first() + if not role: + if auto_create: + return self.create_private_user_role( user ) + else: + return None + return role + def get_sharing_roles( self, user ): + return self.sa_session.query( self.model.Role ) \ + .filter( and_( ( self.model.Role.table.c.name ).like( "Sharing role for: %" + user.email + "%" ), + self.model.Role.table.c.type == self.model.Role.types.SHARING ) ) + def user_set_default_permissions( self, user, permissions={}, history=False, dataset=False, bypass_manage_permission=False, default_access_private = False ): + # bypass_manage_permission is used to change permissions of datasets in a userless history when logging in + flush_needed = False + if user is None: + return None + if not permissions: + #default permissions + permissions = { self.permitted_actions.DATASET_MANAGE_PERMISSIONS : [ self.get_private_user_role( user, auto_create=True ) ] } + #new_user_dataset_access_role_default_private is set as True in config file + if default_access_private: + permissions[ self.permitted_actions.DATASET_ACCESS ] = permissions.values()[ 0 ] + # Delete all of the current default permissions for the user + for dup in user.default_permissions: + self.sa_session.delete( dup ) + flush_needed = True + # Add the new default permissions for the user + for action, roles in permissions.items(): + if isinstance( action, Action ): + action = action.action + for dup in [ self.model.DefaultUserPermissions( user, action, role ) for role in roles ]: + self.sa_session.add( dup ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + if history: + for history in user.active_histories: + self.history_set_default_permissions( history, permissions=permissions, dataset=dataset, bypass_manage_permission=bypass_manage_permission ) + def user_get_default_permissions( self, user ): + permissions = {} + for dup in user.default_permissions: + action = self.get_action( dup.action ) + if action in permissions: + permissions[ action ].append( dup.role ) + else: + permissions[ action ] = [ dup.role ] + return permissions + def history_set_default_permissions( self, history, permissions={}, dataset=False, bypass_manage_permission=False ): + # bypass_manage_permission is used to change permissions of datasets in a user-less history when logging in + flush_needed = False + user = history.user + if not user: + # default permissions on a user-less history are None + return None + if not permissions: + permissions = self.user_get_default_permissions( user ) + # Delete all of the current default permission for the history + for dhp in history.default_permissions: + self.sa_session.delete( dhp ) + flush_needed = True + # Add the new default permissions for the history + for action, roles in permissions.items(): + if isinstance( action, Action ): + action = action.action + for dhp in [ self.model.DefaultHistoryPermissions( history, action, role ) for role in roles ]: + self.sa_session.add( dhp ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + if dataset: + # Only deal with datasets that are not purged + for hda in history.activatable_datasets: + dataset = hda.dataset + if dataset.library_associations: + # Don't change permissions on a dataset associated with a library + continue + if [ assoc for assoc in dataset.history_associations if assoc.history not in user.histories ]: + # Don't change permissions on a dataset associated with a history not owned by the user + continue + if bypass_manage_permission or self.can_manage_dataset( user.all_roles(), dataset ): + self.set_all_dataset_permissions( dataset, permissions ) + def history_get_default_permissions( self, history ): + permissions = {} + for dhp in history.default_permissions: + action = self.get_action( dhp.action ) + if action in permissions: + permissions[ action ].append( dhp.role ) + else: + permissions[ action ] = [ dhp.role ] + return permissions + def set_all_dataset_permissions( self, dataset, permissions={} ): + """ + Set new permissions on a dataset, eliminating all current permissions + permissions looks like: { Action : [ Role, Role ] } + """ + # Make sure that DATASET_MANAGE_PERMISSIONS is associated with at least 1 role + has_dataset_manage_permissions = False + for action, roles in permissions.items(): + if isinstance( action, Action ): + if action == self.permitted_actions.DATASET_MANAGE_PERMISSIONS and roles: + has_dataset_manage_permissions = True + break + elif action == self.permitted_actions.DATASET_MANAGE_PERMISSIONS.action and roles: + has_dataset_manage_permissions = True + break + if not has_dataset_manage_permissions: + return "At least 1 role must be associated with the manage permissions permission on this dataset." + flush_needed = False + # Delete all of the current permissions on the dataset + for dp in dataset.actions: + self.sa_session.delete( dp ) + flush_needed = True + # Add the new permissions on the dataset + for action, roles in permissions.items(): + if isinstance( action, Action ): + action = action.action + for dp in [ self.model.DatasetPermissions( action, dataset, role ) for role in roles ]: + self.sa_session.add( dp ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + return "" + def set_dataset_permission( self, dataset, permission={} ): + """ + Set a specific permission on a dataset, leaving all other current permissions on the dataset alone + permissions looks like: { Action : [ Role, Role ] } + """ + flush_needed = False + for action, roles in permission.items(): + if isinstance( action, Action ): + action = action.action + # Delete the current specific permission on the dataset if one exists + for dp in dataset.actions: + if dp.action == action: + self.sa_session.delete( dp ) + flush_needed = True + # Add the new specific permission on the dataset + for dp in [ self.model.DatasetPermissions( action, dataset, role ) for role in roles ]: + self.sa_session.add( dp ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + def get_permissions( self, item ): + """ + Return a dictionary containing the actions and associated roles on item + where item is one of Library, LibraryFolder, LibraryDatasetDatasetAssociation, + LibraryDataset, Dataset. The dictionary looks like: { Action : [ Role, Role ] }. + """ + permissions = {} + for item_permission in item.actions: + action = self.get_action( item_permission.action ) + if action in permissions: + permissions[ action ].append( item_permission.role ) + else: + permissions[ action ] = [ item_permission.role ] + return permissions + def get_accessible_request_types( self, trans, user ): + """Return all RequestTypes that the received user has permission to access.""" + accessible_request_types = [] + current_user_role_ids = [ role.id for role in user.all_roles() ] + request_type_access_action = self.permitted_actions.REQUEST_TYPE_ACCESS.action + restricted_request_type_ids = [ rtp.request_type_id for rtp in trans.sa_session.query( trans.model.RequestTypePermissions ) \ + .filter( trans.model.RequestTypePermissions.table.c.action == request_type_access_action ) \ + .distinct() ] + accessible_restricted_request_type_ids = [ rtp.request_type_id for rtp in trans.sa_session.query( trans.model.RequestTypePermissions ) \ + .filter( and_( trans.model.RequestTypePermissions.table.c.action == request_type_access_action, + trans.model.RequestTypePermissions.table.c.role_id.in_( current_user_role_ids ) ) ) ] + # Filter to get libraries accessible by the current user. Get both + # public libraries and restricted libraries accessible by the current user. + for request_type in trans.sa_session.query( trans.model.RequestType ) \ + .filter( and_( trans.model.RequestType.table.c.deleted == False, + ( or_( not_( trans.model.RequestType.table.c.id.in_( restricted_request_type_ids ) ), + trans.model.RequestType.table.c.id.in_( accessible_restricted_request_type_ids ) ) ) ) ) \ + .order_by( trans.app.model.RequestType.name ): + accessible_request_types.append( request_type ) + return accessible_request_types + def copy_dataset_permissions( self, src, dst ): + if not isinstance( src, self.model.Dataset ): + src = src.dataset + if not isinstance( dst, self.model.Dataset ): + dst = dst.dataset + self.set_all_dataset_permissions( dst, self.get_permissions( src ) ) + def privately_share_dataset( self, dataset, users = [] ): + intersect = None + for user in users: + roles = [ ura.role for ura in user.roles if ura.role.type == self.model.Role.types.SHARING ] + if intersect is None: + intersect = roles + else: + new_intersect = [] + for role in roles: + if role in intersect: + new_intersect.append( role ) + intersect = new_intersect + sharing_role = None + if intersect: + for role in intersect: + if not filter( lambda x: x not in users, [ ura.user for ura in role.users ] ): + # only use a role if it contains ONLY the users we're sharing with + sharing_role = role + break + if sharing_role is None: + sharing_role = self.model.Role( name = "Sharing role for: " + ", ".join( [ u.email for u in users ] ), + type = self.model.Role.types.SHARING ) + self.sa_session.add( sharing_role ) + self.sa_session.flush() + for user in users: + self.associate_components( user=user, role=sharing_role ) + self.set_dataset_permission( dataset, { self.permitted_actions.DATASET_ACCESS : [ sharing_role ] } ) + def set_all_library_permissions( self, trans, library_item, permissions={} ): + # Set new permissions on library_item, eliminating all current permissions + flush_needed = False + for role_assoc in library_item.actions: + self.sa_session.delete( role_assoc ) + flush_needed = True + # Add the new permissions on library_item + for item_class, permission_class in self.library_item_assocs: + if isinstance( library_item, item_class ): + for action, roles in permissions.items(): + if isinstance( action, Action ): + action = action.action + for role_assoc in [ permission_class( action, library_item, role ) for role in roles ]: + self.sa_session.add( role_assoc ) + flush_needed = True + if isinstance( library_item, self.model.LibraryDatasetDatasetAssociation ): + # Permission setting related to DATASET_MANAGE_PERMISSIONS was broken for a period of time, + # so it is possible that some Datasets have no roles associated with the DATASET_MANAGE_PERMISSIONS + # permission. In this case, we'll reset this permission to the library_item user's private role. + if not library_item.dataset.has_manage_permissions_roles( trans ): + permission = {} + permissions[ self.permitted_actions.DATASET_MANAGE_PERMISSIONS ] = [ trans.app.security_agent.get_private_user_role( library_item.user ) ] + self.set_dataset_permission( library_item.dataset, permissions ) + if action == self.permitted_actions.LIBRARY_MANAGE.action and roles: + # Handle the special case when we are setting the LIBRARY_MANAGE_PERMISSION on a + # library_dataset_dataset_association since the roles need to be applied to the + # DATASET_MANAGE_PERMISSIONS permission on the associated dataset. + permissions = {} + permissions[ self.permitted_actions.DATASET_MANAGE_PERMISSIONS ] = roles + self.set_dataset_permission( library_item.dataset, permissions ) + if flush_needed: + self.sa_session.flush() + def library_is_public( self, library, contents=False ): + if contents: + # Check all contained folders and datasets to find any that are not public + if not self.folder_is_public( library.root_folder ): + return False + # A library is considered public if there are no "access" actions associated with it. + return self.permitted_actions.LIBRARY_ACCESS.action not in [ a.action for a in library.actions ] + def make_library_public( self, library, contents=False ): + flush_needed = False + if contents: + # Make all contained folders (include deleted folders, but not purged folders), public + self.make_folder_public( library.root_folder ) + # A library is considered public if there are no LIBRARY_ACCESS actions associated with it. + for lp in library.actions: + if lp.action == self.permitted_actions.LIBRARY_ACCESS.action: + self.sa_session.delete( lp ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + def folder_is_public( self, folder ): + for sub_folder in folder.folders: + if not self.folder_is_public( sub_folder ): + return False + for library_dataset in folder.datasets: + ldda = library_dataset.library_dataset_dataset_association + if ldda and ldda.dataset and not self.dataset_is_public( ldda.dataset ): + return False + return True + def make_folder_public( self, folder ): + # Make all of the contents (include deleted contents, but not purged contents) of folder public + for sub_folder in folder.folders: + if not sub_folder.purged: + self.make_folder_public( sub_folder ) + for library_dataset in folder.datasets: + dataset = library_dataset.library_dataset_dataset_association.dataset + if not dataset.purged and not self.dataset_is_public( dataset ): + self.make_dataset_public( dataset ) + + def dataset_is_public( self, dataset ): + # A dataset is considered public if there are no "access" actions associated with it. Any + # other actions ( 'manage permissions', 'edit metadata' ) are irrelevant. + # SM: Accessing dataset.actions will cause a query to be emitted. + return self.permitted_actions.DATASET_ACCESS.action not in [ a.action for a in dataset.actions ] + + def datasets_are_public( self, trans, datasets ): + ''' + Given a transaction object and a list of Datasets, return + a mapping from Dataset ids to whether the Dataset is public + or not. All Dataset ids should be returned in the mapping's keys. + ''' + # We go the other way around from dataset_is_public: we start with + # all datasets being marked as public. If there is an access action + # associated with the dataset, then we mark it as nonpublic: + datasets_public = {} + dataset_ids = [ dataset.id for dataset in datasets ] + for dataset_id in dataset_ids: + datasets_public[ dataset_id ] = True + + # Now get all datasets which have DATASET_ACCESS actions: + access_data_perms = trans.sa_session.query( trans.app.model.DatasetPermissions ) \ + .filter( and_( trans.app.model.DatasetPermissions.dataset_id.in_( dataset_ids ), + trans.app.model.DatasetPermissions.action == self.permitted_actions.DATASET_ACCESS.action ) ) \ + .all() + # Every dataset returned has "access" privileges associated with it, + # so it's not public. + for permission in access_data_perms: + datasets_public[ permission.dataset_id ] = False + return datasets_public + + + def make_dataset_public( self, dataset ): + # A dataset is considered public if there are no "access" actions associated with it. Any + # other actions ( 'manage permissions', 'edit metadata' ) are irrelevant. + flush_needed = False + for dp in dataset.actions: + if dp.action == self.permitted_actions.DATASET_ACCESS.action: + self.sa_session.delete( dp ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + def derive_roles_from_access( self, trans, item_id, cntrller, library=False, **kwd ): + # Check the access permission on a dataset. If library is true, item_id refers to a library. If library + # is False, item_id refers to a dataset ( item_id must currently be decoded before being sent ). The + # cntrller param is the calling controller, which needs to be passed to get_legitimate_roles(). + msg = '' + permissions = {} + # accessible will be True only if at least 1 user has every role in DATASET_ACCESS_in + accessible = False + # legitimate will be True only if all roles in DATASET_ACCESS_in are in the set of roles returned from + # get_legitimate_roles() + legitimate = False + # private_role_found will be true only if more than 1 role is being associated with the DATASET_ACCESS + # permission on item, and at least 1 of the roles is private. + private_role_found = False + error = False + for k, v in get_permitted_actions( filter='DATASET' ).items(): + in_roles = [ self.sa_session.query( self.model.Role ).get( x ) for x in listify( kwd.get( k + '_in', [] ) ) ] + if v == self.permitted_actions.DATASET_ACCESS and in_roles: + if library: + item = self.sa_session.query( self.model.Library ).get( item_id ) + else: + item = self.sa_session.query( self.model.Dataset ).get( item_id ) + if ( library and not self.library_is_public( item ) ) or ( not library and not self.dataset_is_public( item ) ): + # Ensure that roles being associated with DATASET_ACCESS are a subset of the legitimate roles + # derived from the roles associated with the access permission on item if it's not public. This + # will keep ill-legitimate roles from being associated with the DATASET_ACCESS permission on the + # dataset (i.e., in the case where item is .a library, if Role1 is associated with LIBRARY_ACCESS, + # then only those users that have Role1 should be associated with DATASET_ACCESS. + legitimate_roles = self.get_legitimate_roles( trans, item, cntrller ) + ill_legitimate_roles = [] + for role in in_roles: + if role not in legitimate_roles: + ill_legitimate_roles.append( role ) + if ill_legitimate_roles: + # This condition should never occur since ill-legitimate roles are filtered out of the set of + # roles displayed on the forms, but just in case there is a bug somewhere that incorrectly + # filters, we'll display this message. + error = True + msg += "The following roles are not associated with users that have the 'access' permission on this " + msg += "item, so they were incorrectly displayed: " + for role in ill_legitimate_roles: + msg += "%s, " % role.name + msg = msg.rstrip( ", " ) + new_in_roles = [] + for role in in_roles: + if role in legitimate_roles: + new_in_roles.append( role ) + in_roles = new_in_roles + else: + legitimate = True + if len( in_roles ) > 1: + # At least 1 user must have every role associated with the access + # permission on this dataset, or the dataset is not accessible. + # Since we have more than 1 role, none of them can be private. + for role in in_roles: + if role.type == self.model.Role.types.PRIVATE: + private_role_found = True + break + if len( in_roles ) == 1: + accessible = True + else: + # At least 1 user must have every role associated with the access + # permission on this dataset, or the dataset is not accessible. + in_roles_set = set() + for role in in_roles: + in_roles_set.add( role ) + users_set = set() + for role in in_roles: + for ura in role.users: + users_set.add( ura.user ) + for gra in role.groups: + group = gra.group + for uga in group.users: + users_set.add( uga.user ) + # Make sure that at least 1 user has every role being associated with the dataset. + for user in users_set: + user_roles_set = set() + for ura in user.roles: + user_roles_set.add( ura.role ) + if in_roles_set.issubset( user_roles_set ): + accessible = True + break + if private_role_found or not accessible: + error = True + # Don't set the permissions for DATASET_ACCESS if inaccessible or multiple roles with + # at least 1 private, but set all other permissions. + permissions[ self.get_action( v.action ) ] = [] + msg = "At least 1 user must have every role associated with accessing datasets. " + if private_role_found: + msg += "Since you are associating more than 1 role, no private roles are allowed." + if not accessible: + msg += "The roles you attempted to associate for access would make the datasets in-accessible by everyone." + else: + permissions[ self.get_action( v.action ) ] = in_roles + else: + permissions[ self.get_action( v.action ) ] = in_roles + return permissions, in_roles, error, msg + def copy_library_permissions( self, trans, source_library_item, target_library_item, user=None ): + # Copy all relevant permissions from source. + permissions = {} + for role_assoc in source_library_item.actions: + if role_assoc.action != self.permitted_actions.LIBRARY_ACCESS.action: + # LIBRARY_ACCESS is a special permission that is set only at the library level. + if role_assoc.action in permissions: + permissions[role_assoc.action].append( role_assoc.role ) + else: + permissions[role_assoc.action] = [ role_assoc.role ] + self.set_all_library_permissions( trans, target_library_item, permissions ) + if user: + item_class = None + for item_class, permission_class in self.library_item_assocs: + if isinstance( target_library_item, item_class ): + break + if item_class: + # Make sure user's private role is included + private_role = self.model.security_agent.get_private_user_role( user ) + for name, action in self.permitted_actions.items(): + if not permission_class.filter_by( role_id = private_role.id, action = action.action ).first(): + lp = permission_class( action.action, target_library_item, private_role ) + self.sa_session.add( lp ) + self.sa_session.flush() + else: + raise 'Invalid class (%s) specified for target_library_item (%s)' % \ + ( target_library_item.__class__, target_library_item.__class__.__name__ ) + def get_permitted_libraries( self, trans, user, actions ): + """ + This method is historical (it is not currently used), but may be useful again at some + point. It returns a dictionary whose keys are library objects and whose values are a + comma-separated string of folder ids. This method works with the show_library_item() + method below, and it returns libraries for which the received user has permission to + perform the received actions. Here is an example call to this method to return all + libraries for which the received user has LIBRARY_ADD permission:: + + libraries = trans.app.security_agent.get_permitted_libraries( trans, user, + [ trans.app.security_agent.permitted_actions.LIBRARY_ADD ] ) + """ + all_libraries = trans.sa_session.query( trans.app.model.Library ) \ + .filter( trans.app.model.Library.table.c.deleted == False ) \ + .order_by( trans.app.model.Library.name ) + roles = user.all_roles() + actions_to_check = actions + # The libraries dictionary looks like: { library : '1,2' }, library : '3' } + # Its keys are the libraries that should be displayed for the current user and whose values are a + # string of comma-separated folder ids, of the associated folders the should NOT be displayed. + # The folders that should not be displayed may not be a complete list, but it is ultimately passed + # to the calling method to keep from re-checking the same folders when the library / folder + # select lists are rendered. + libraries = {} + for library in all_libraries: + can_show, hidden_folder_ids = self.show_library_item( self, roles, library, actions_to_check ) + if can_show: + libraries[ library ] = hidden_folder_ids + return libraries + def show_library_item( self, user, roles, library_item, actions_to_check, hidden_folder_ids='' ): + """ + This method must be sent an instance of Library() or LibraryFolder(). Recursive execution produces a + comma-separated string of folder ids whose folders do NOT meet the criteria for showing. Along with + the string, True is returned if the current user has permission to perform any 1 of actions_to_check + on library_item. Otherwise, cycle through all sub-folders in library_item until one is found that meets + this criteria, if it exists. This method does not necessarily scan the entire library as it returns + when it finds the first library_item that allows user to perform any one action in actions_to_check. + """ + for action in actions_to_check: + if self.allow_action( roles, action, library_item ): + return True, hidden_folder_ids + if isinstance( library_item, self.model.Library ): + return self.show_library_item( user, roles, library_item.root_folder, actions_to_check, hidden_folder_ids='' ) + if isinstance( library_item, self.model.LibraryFolder ): + for folder in library_item.active_folders: + can_show, hidden_folder_ids = self.show_library_item( user, roles, folder, actions_to_check, hidden_folder_ids=hidden_folder_ids ) + if can_show: + return True, hidden_folder_ids + if hidden_folder_ids: + hidden_folder_ids = '%s,%d' % ( hidden_folder_ids, folder.id ) + else: + hidden_folder_ids = '%d' % folder.id + return False, hidden_folder_ids + def get_showable_folders( self, user, roles, library_item, actions_to_check, hidden_folder_ids=[], showable_folders=[] ): + """ + This method must be sent an instance of Library(), all the folders of which are scanned to determine if + user is allowed to perform any action in actions_to_check. The param hidden_folder_ids, if passed, should + contain a list of folder IDs which was generated when the library was previously scanned + using the same actions_to_check. A list of showable folders is generated. This method scans the entire library. + """ + if isinstance( library_item, self.model.Library ): + return self.get_showable_folders( user, roles, library_item.root_folder, actions_to_check, showable_folders=[] ) + if isinstance( library_item, self.model.LibraryFolder ): + if library_item.id not in hidden_folder_ids: + for action in actions_to_check: + if self.allow_action( roles, action, library_item ): + showable_folders.append( library_item ) + break + for folder in library_item.active_folders: + self.get_showable_folders( user, roles, folder, actions_to_check, showable_folders=showable_folders ) + return showable_folders + def set_entity_user_associations( self, users=[], roles=[], groups=[], delete_existing_assocs=True ): + for user in users: + if delete_existing_assocs: + flush_needed = False + for a in user.non_private_roles + user.groups: + self.sa_session.delete( a ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + self.sa_session.refresh( user ) + for role in roles: + # Make sure we are not creating an additional association with a PRIVATE role + if role not in user.roles: + self.associate_components( user=user, role=role ) + for group in groups: + self.associate_components( user=user, group=group ) + def set_entity_group_associations( self, groups=[], users=[], roles=[], delete_existing_assocs=True ): + for group in groups: + if delete_existing_assocs: + flush_needed = False + for a in group.roles + group.users: + self.sa_session.delete( a ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + for role in roles: + self.associate_components( group=group, role=role ) + for user in users: + self.associate_components( group=group, user=user ) + def set_entity_role_associations( self, roles=[], users=[], groups=[], delete_existing_assocs=True ): + for role in roles: + if delete_existing_assocs: + flush_needed = False + for a in role.users + role.groups: + self.sa_session.delete( a ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + for user in users: + self.associate_components( user=user, role=role ) + for group in groups: + self.associate_components( group=group, role=role ) + def get_component_associations( self, **kwd ): + assert len( kwd ) == 2, 'You must specify exactly 2 Galaxy security components to check for associations.' + if 'dataset' in kwd: + if 'action' in kwd: + return self.sa_session.query( self.model.DatasetPermissions ).filter_by( action = kwd['action'].action, dataset_id = kwd['dataset'].id ).first() + elif 'user' in kwd: + if 'group' in kwd: + return self.sa_session.query( self.model.UserGroupAssociation ).filter_by( group_id = kwd['group'].id, user_id = kwd['user'].id ).first() + elif 'role' in kwd: + return self.sa_session.query( self.model.UserRoleAssociation ).filter_by( role_id = kwd['role'].id, user_id = kwd['user'].id ).first() + elif 'group' in kwd: + if 'role' in kwd: + return self.sa_session.query( self.model.GroupRoleAssociation ).filter_by( role_id = kwd['role'].id, group_id = kwd['group'].id ).first() + raise 'No valid method of associating provided components: %s' % kwd + def check_folder_contents( self, user, roles, folder, hidden_folder_ids='' ): + """ + This method must always be sent an instance of LibraryFolder(). Recursive execution produces a + comma-separated string of folder ids whose folders do NOT meet the criteria for showing. Along + with the string, True is returned if the current user has permission to access folder. Otherwise, + cycle through all sub-folders in folder until one is found that meets this criteria, if it exists. + This method does not necessarily scan the entire library as it returns when it finds the first + folder that is accessible to user. + """ + # If a folder is writeable, it's accessable and we need not go further + if self.can_add_library_item( roles, folder ): + return True, '' + action = self.permitted_actions.DATASET_ACCESS + + lddas = self.sa_session.query( self.model.LibraryDatasetDatasetAssociation ) \ + .join( "library_dataset" ) \ + .filter( self.model.LibraryDataset.folder == folder ) \ + .join( "dataset" ) \ + .options( eagerload_all( "dataset.actions" ) ) \ + .all() + + for ldda in lddas: + ldda_access_permissions = self.get_item_actions( action, ldda.dataset ) + if not ldda_access_permissions: + # Dataset is public + return True, hidden_folder_ids + for ldda_access_permission in ldda_access_permissions: + if ldda_access_permission.role in roles: + # The current user has access permission on the dataset + return True, hidden_folder_ids + for sub_folder in folder.active_folders: + can_access, hidden_folder_ids = self.check_folder_contents( user, roles, sub_folder, hidden_folder_ids=hidden_folder_ids ) + if can_access: + return True, hidden_folder_ids + if hidden_folder_ids: + hidden_folder_ids = '%s,%d' % ( hidden_folder_ids, sub_folder.id ) + else: + hidden_folder_ids = '%d' % sub_folder.id + return False, hidden_folder_ids + def can_access_request_type( self, roles, request_type ): + action = self.permitted_actions.REQUEST_TYPE_ACCESS + request_type_actions = [] + for permission in request_type.actions: + if permission.action == action.action: + request_type_actions.append( permission ) + if not request_type_actions: + return True + ret_val = False + for request_type_action in request_type_actions: + if request_type_action.role in roles: + ret_val = True + break + return ret_val + def set_request_type_permissions( self, request_type, permissions={} ): + # Set new permissions on request_type, eliminating all current permissions + for role_assoc in request_type.actions: + self.sa_session.delete( role_assoc ) + # Add the new permissions on request_type + item_class = self.model.RequestType + permission_class = self.model.RequestTypePermissions + flush_needed = False + for action, roles in permissions.items(): + if isinstance( action, Action ): + action = action.action + for role_assoc in [ permission_class( action, request_type, role ) for role in roles ]: + self.sa_session.add( role_assoc ) + flush_needed = True + if flush_needed: + self.sa_session.flush() + +class HostAgent( RBACAgent ): + """ + A simple security agent which allows access to datasets based on host. + This exists so that externals sites such as UCSC can gain access to + datasets which have permissions which would normally prevent such access. + """ + # TODO: Make sites user configurable + sites = Bunch( + ucsc_main = ( 'hgw1.cse.ucsc.edu', 'hgw2.cse.ucsc.edu', 'hgw3.cse.ucsc.edu', 'hgw4.cse.ucsc.edu', + 'hgw5.cse.ucsc.edu', 'hgw6.cse.ucsc.edu', 'hgw7.cse.ucsc.edu', 'hgw8.cse.ucsc.edu' ), + ucsc_test = ( 'hgwdev.cse.ucsc.edu', ), + ucsc_archaea = ( 'lowepub.cse.ucsc.edu', ) + ) + def __init__( self, model, permitted_actions=None ): + self.model = model + if permitted_actions: + self.permitted_actions = permitted_actions + @property + def sa_session( self ): + """Returns a SQLAlchemy session""" + return self.model.context + def allow_action( self, addr, action, **kwd ): + if 'dataset' in kwd and action == self.permitted_actions.DATASET_ACCESS: + hda = kwd['dataset'] + if action == self.permitted_actions.DATASET_ACCESS and action.action not in [ dp.action for dp in hda.dataset.actions ]: + log.debug( 'Allowing access to public dataset with hda: %i.' % hda.id ) + return True # dataset has no roles associated with the access permission, thus is already public + hdadaa = self.sa_session.query( self.model.HistoryDatasetAssociationDisplayAtAuthorization ) \ + .filter_by( history_dataset_association_id = hda.id ).first() + if not hdadaa: + log.debug( 'Denying access to private dataset with hda: %i. No hdadaa record for this dataset.' % hda.id ) + return False # no auth + # We could just look up the reverse of addr, but then we'd also + # have to verify it with the forward address and special case any + # IPs (instead of hosts) in the server list. + # + # This would be improved by caching, but that's what the OS's name + # service cache daemon is for (you ARE running nscd, right?). + for server in HostAgent.sites.get( hdadaa.site, [] ): + # We're going to search in order, but if the remote site is load + # balancing their connections (as UCSC does), this is okay. + try: + if socket.gethostbyname( server ) == addr: + break # remote host is in the server list + except ( socket.error, socket.gaierror ): + pass # can't resolve, try next + else: + log.debug( 'Denying access to private dataset with hda: %i. Remote addr is not a valid server for site: %s.' % ( hda.id, hdadaa.site ) ) + return False # remote addr is not in the server list + if ( datetime.utcnow() - hdadaa.update_time ) > timedelta( seconds=60 ): + log.debug( 'Denying access to private dataset with hda: %i. Authorization was granted, but has expired.' % hda.id ) + return False # not authz'd in the last 60 seconds + log.debug( 'Allowing access to private dataset with hda: %i. Remote server is: %s.' % ( hda.id, server ) ) + return True + else: + raise 'The dataset access permission is the only valid permission in the host security agent.' + def set_dataset_permissions( self, hda, user, site ): + hdadaa = self.sa_session.query( self.model.HistoryDatasetAssociationDisplayAtAuthorization ) \ + .filter_by( history_dataset_association_id = hda.id ).first() + if hdadaa: + hdadaa.update_time = datetime.utcnow() + else: + hdadaa = self.model.HistoryDatasetAssociationDisplayAtAuthorization( hda=hda, user=user, site=site ) + self.sa_session.add( hdadaa ) + self.sa_session.flush() + +def get_permitted_actions( filter=None ): + '''Utility method to return a subset of RBACAgent's permitted actions''' + if filter is None: + return RBACAgent.permitted_actions + tmp_bunch = Bunch() + [ tmp_bunch.__dict__.__setitem__(k, v) for k, v in RBACAgent.permitted_actions.items() if k.startswith( filter ) ] + return tmp_bunch diff --git a/lib/galaxy/security/passwords.py b/lib/galaxy/security/passwords.py index ae4f2476d0f..bdcc2514738 100644 --- a/lib/galaxy/security/passwords.py +++ b/lib/galaxy/security/passwords.py @@ -19,7 +19,7 @@ def hash_password( password ): def check_password( guess, hashed ): """ - Check a hashed password. Supports either PBKDF2 if the hash is + Check a hashed password. Supports either PBKDF2 if the hash is prefixed with that string, or sha1 otherwise. """ if hashed.startswith( "PBKDF2" ): diff --git a/lib/galaxy/security/validate_user_input.py b/lib/galaxy/security/validate_user_input.py index 705519df9ff..e6ce72a904b 100644 --- a/lib/galaxy/security/validate_user_input.py +++ b/lib/galaxy/security/validate_user_input.py @@ -7,7 +7,7 @@ FILL_CHAR = '-' def validate_email( trans, email, user=None, check_dup=True ): message = '' if user and user.email == email: - return message + return message if len( email ) == 0 or "@" not in email or "." not in email: message = "Enter a real email address" elif len( email ) > 255: @@ -28,7 +28,7 @@ def validate_publicname( trans, publicname, user=None ): return "Public name must be at least 3 characters in length" else: if len( publicname ) < 4: - return "Public name must be at least 4 characters in length" + return "Public name must be at least 4 characters in length" if len( publicname ) > 255: return "Public name cannot be more than 255 characters in length" if not( VALID_PUBLICNAME_RE.match( publicname ) ): diff --git a/lib/galaxy/tags/tag_handler.py b/lib/galaxy/tags/tag_handler.py index cd2ca5d9737..02ae6aa0e63 100644 --- a/lib/galaxy/tags/tag_handler.py +++ b/lib/galaxy/tags/tag_handler.py @@ -38,8 +38,8 @@ class TagHandler( object ): item_tag_assoc_class = self.get_tag_assoc_class( item_class ) if not item_tag_assoc_class: return [] - # Build select statement. - cols_to_select = [ item_tag_assoc_class.table.c.tag_id, func.count( '*' ) ] + # Build select statement. + cols_to_select = [ item_tag_assoc_class.table.c.tag_id, func.count( '*' ) ] from_obj = item_tag_assoc_class.table.join( item_class.table ).join( trans.app.model.Tag.table ) where_clause = ( self.get_id_col_in_item_tag_assoc_table( item_class ) == item.id ) order_by = [ func.count( "*" ).desc() ] @@ -56,7 +56,7 @@ class TagHandler( object ): community_tags = [] for row in result_set: tag_id = row[0] - community_tags.append( self.get_tag_by_id( trans, tag_id ) ) + community_tags.append( self.get_tag_by_id( trans, tag_id ) ) return community_tags def get_tool_tags( self, trans ): result_set = trans.sa_session.execute( select( columns=[ trans.app.model.ToolTagAssociation.table.c.tag_id ], @@ -115,7 +115,7 @@ class TagHandler( object ): # Add tag to association. item.tags.append( item_tag_assoc ) item_tag_assoc.tag = tag - item_tag_assoc.user = user + item_tag_assoc.user = user # Apply attributes to item-tag association. Strip whitespace from user name and tag. lc_value = None if value: @@ -146,7 +146,7 @@ class TagHandler( object ): return ", ".join( tags_str_list ) def get_tag_by_id( self, trans, tag_id ): """Get a Tag object from a tag id.""" - return trans.sa_session.query( trans.app.model.Tag ).filter_by( id=tag_id ).first() + return trans.sa_session.query( trans.app.model.Tag ).filter_by( id=tag_id ).first() def get_tag_by_name( self, trans, tag_name ): """Get a Tag object from a tag name (string).""" if tag_name: @@ -189,11 +189,11 @@ class TagHandler( object ): scrubbed_tag_name = self._scrub_tag_name( tag_name ) for item_tag_assoc in item.tags: if ( item_tag_assoc.user == user ) and ( item_tag_assoc.user_tname == scrubbed_tag_name ): - return item_tag_assoc + return item_tag_assoc return None def parse_tags( self, tag_str ): """ - Returns a list of raw (tag-name, value) pairs derived from a string; method scrubs tag names and values as well. + Returns a list of raw (tag-name, value) pairs derived from a string; method scrubs tag names and values as well. Return value is a dictionary where tag-names are keys. """ # Gracefully handle None. diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 31cf3f05703..057a033e18a 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -450,7 +450,7 @@ class ToolBox( object, DictifiableMixin ): tool_version_select_field = self.build_tool_version_select_field( tools, tool.id, set_selected ) break return tool_version_select_field, tools, tool - + def build_tool_version_select_field( self, tools, tool_id, set_selected ): """Build a SelectField whose options are the ids for the received list of tools.""" options = [] @@ -466,7 +466,7 @@ class ToolBox( object, DictifiableMixin ): else: select_field.add_option( 'version %s' % option_tup[0], option_tup[1] ) return select_field - + def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): try: path = elem.get( "file" ) @@ -659,7 +659,7 @@ class ToolBox( object, DictifiableMixin ): message += "version: %s" % old_tool.version status = 'done' return message, status - + def remove_tool_by_id( self, tool_id ): """ Attempt to remove the tool identified by 'tool_id'. @@ -688,7 +688,7 @@ class ToolBox( object, DictifiableMixin ): message += "version: %s" % tool.version status = 'done' return message, status - + def load_workflow( self, workflow_id ): """ Return an instance of 'Workflow' identified by `id`, @@ -697,7 +697,7 @@ class ToolBox( object, DictifiableMixin ): id = self.app.security.decode_id( workflow_id ) stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id ) return stored.latest_workflow - + def init_dependency_manager( self ): if self.app.config.use_tool_dependencies: self.dependency_manager = DependencyManager( [ self.app.config.tool_dependency_dir ] ) @@ -1566,7 +1566,7 @@ class Tool( object, DictifiableMixin ): elif ( re.search( "fatal", err_level, re.IGNORECASE ) ): return_level = StdioErrorLevel.FATAL else: - log.debug( "Tool %s: error level %s did not match log/warning/fatal" % + log.debug( "Tool %s: error level %s did not match log/warning/fatal" % ( self.id, err_level ) ) except Exception: log.error( "Exception in parse_error_level " @@ -1823,7 +1823,7 @@ class Tool( object, DictifiableMixin ): version = requirement_elem.get( "version", None ) requirement = ToolRequirement( name=name, type=type, version=version ) self.requirements.append( requirement ) - + def populate_tool_shed_info( self ): if self.repository_id is not None and 'ToolShedRepository' in self.app.model: repository_id = self.app.security.decode_id( self.repository_id ) @@ -1833,7 +1833,7 @@ class Tool( object, DictifiableMixin ): self.repository_name = tool_shed_repository.name self.repository_owner = tool_shed_repository.owner self.installed_changeset_revision = tool_shed_repository.installed_changeset_revision - + def check_workflow_compatible( self, root ): """ Determine if a tool can be used in workflows. External tools and the @@ -2967,7 +2967,7 @@ class Tool( object, DictifiableMixin ): # Basic information tool_dict = super( Tool, self ).dictify() - + # Add link details. if link_details: # Add details for creating a hyperlink to the tool. @@ -3223,28 +3223,28 @@ for tool_class in [ Tool, DataDestinationTool, SetMetadataTool, DataSourceTool, class TracksterConfig: """ Trackster configuration encapsulation. """ - + def __init__( self, actions ): self.actions = actions - + @staticmethod def parse( root ): actions = [] for action_elt in root.findall( "action" ): actions.append( SetParamAction.parse( action_elt ) ) return TracksterConfig( actions ) - + class SetParamAction: """ Set parameter action. """ - + def __init__( self, name, output_name ): self.name = name self.output_name = output_name - + @staticmethod def parse( elt ): """ Parse action from element. """ - return SetParamAction( elt.get( "name" ), elt.get( "output_name" ) ) + return SetParamAction( elt.get( "name" ), elt.get( "output_name" ) ) class BadValue( object ): def __init__( self, value ): @@ -3305,7 +3305,7 @@ class RawObjectWrapper( ToolParameterValueWrapper ): try: return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) except: - #Most likely None, which lacks __module__. + #Most likely None, which lacks __module__. return str( self.obj ) def __getattr__( self, key ): return getattr( self.obj, key ) diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 70503797459..28c84b7d03d 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -22,13 +22,13 @@ class ToolAction( object ): """ def execute( self, tool, trans, incoming={}, set_output_hid=True ): raise TypeError("Abstract method") - + class DefaultToolAction( object ): """Default tool action is to run an external command""" - + def collect_input_datasets( self, tool, param_values, trans ): """ - Collect any dataset inputs from incoming. Returns a mapping from + Collect any dataset inputs from incoming. Returns a mapping from parameter name to Dataset instance for each tool parameter that is of the DataToolParameter type. """ @@ -117,9 +117,9 @@ class DefaultToolAction( object ): def make_dict_copy( from_dict ): """ Makes a copy of input dictionary from_dict such that all values that are dictionaries - result in creation of a new dictionary ( a sort of deepcopy ). We may need to handle - other complex types ( e.g., lists, etc ), but not sure... - Yes, we need to handle lists (and now are)... + result in creation of a new dictionary ( a sort of deepcopy ). We may need to handle + other complex types ( e.g., lists, etc ), but not sure... + Yes, we need to handle lists (and now are)... """ copy_from_dict = {} for key, value in from_dict.items(): @@ -168,11 +168,11 @@ class DefaultToolAction( object ): input_values[ input.name ] = galaxy.tools.SelectToolParameterWrapper( input, input_values[ input.name ], tool.app, other_values = incoming ) else: input_values[ input.name ] = galaxy.tools.InputValueWrapper( input, input_values[ input.name ], incoming ) - + # Set history. if not history: history = tool.get_default_history_by_trans( trans, create=True ) - + out_data = odict() # Collect any input datasets from the incoming parameters inp_data = self.collect_input_datasets( tool, incoming, trans ) @@ -185,17 +185,17 @@ class DefaultToolAction( object ): if not data: data = NoneDataset( datatypes_registry = trans.app.datatypes_registry ) continue - + # Convert LDDA to an HDA. if isinstance(data, LibraryDatasetDatasetAssociation): data = data.to_history_dataset_association( None ) inp_data[name] = data - + else: # HDA if data.hid: input_names.append( 'data %s' % data.hid ) input_ext = data.ext - + if data.dbkey not in [None, '?']: input_dbkey = data.dbkey @@ -214,13 +214,13 @@ class DefaultToolAction( object ): if 'fasta' in custom_build_dict: build_fasta_dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( custom_build_dict[ 'fasta' ] ) chrom_info = build_fasta_dataset.get_converted_dataset( trans, 'len' ).file_name - + if not chrom_info: # Default to built-in build. chrom_info = os.path.join( trans.app.config.len_file_path, "%s.len" % input_dbkey ) incoming[ "chromInfo" ] = chrom_info inp_data.update( db_datasets ) - + # Determine output dataset permission/roles list existing_datasets = [ inp for inp in inp_data.values() if inp ] if existing_datasets: @@ -242,7 +242,7 @@ class DefaultToolAction( object ): # Add the dbkey to the incoming parameters incoming[ "dbkey" ] = input_dbkey params = None #wrapped params are used by change_format action and by output.label; only perform this wrapping once, as needed - # Keep track of parent / child relationships, we'll create all the + # Keep track of parent / child relationships, we'll create all the # datasets first, then create the associations parent_to_child_pairs = [] child_dataset_names = set() @@ -258,7 +258,7 @@ class DefaultToolAction( object ): if output.parent: parent_to_child_pairs.append( ( output.parent, name ) ) child_dataset_names.add( name ) - ## What is the following hack for? Need to document under what + ## What is the following hack for? Need to document under what ## conditions can the following occur? (james@bx.psu.edu) # HACK: the output data has already been created # this happens i.e. as a result of the async controller @@ -279,7 +279,7 @@ class DefaultToolAction( object ): ext = input_extension except Exception, e: pass - + #process change_format tags if output.change_format: if params is None: @@ -322,14 +322,14 @@ class DefaultToolAction( object ): object_store_id = data.dataset.object_store_id # these will be the same thing after the first output # This may not be neccesary with the new parent/child associations data.designation = name - # Copy metadata from one of the inputs if requested. + # Copy metadata from one of the inputs if requested. if output.metadata_source: data.init_meta( copy_from=inp_data[output.metadata_source] ) else: data.init_meta() # Take dbkey from LAST input data.dbkey = str(input_dbkey) - # Set state + # Set state # FIXME: shouldn't this be NEW until the job runner changes it? data.state = data.states.QUEUED data.blurb = "queued" @@ -351,7 +351,7 @@ class DefaultToolAction( object ): params = make_dict_copy( incoming ) wrap_values( tool.inputs, params, skip_missing_values = not tool.check_values ) data.name = self._get_default_data_name( data, tool, on_text=on_text, trans=trans, incoming=incoming, history=history, params=params, job_params=job_params ) - # Store output + # Store output out_data[ name ] = data if output.actions: #Apply pre-job tool-output-dataset actions; e.g. setting metadata, changing format @@ -373,7 +373,7 @@ class DefaultToolAction( object ): parent_dataset = out_data[ parent_name ] child_dataset = out_data[ child_name ] parent_dataset.children.append( child_dataset ) - # Store data after custom code runs + # Store data after custom code runs trans.sa_session.flush() # Create the job object job = trans.app.model.Job() @@ -454,7 +454,7 @@ class DefaultToolAction( object ): for name in inp_data.keys(): dataset = inp_data[ name ] redirect_url = tool.parse_redirect_url( dataset, incoming ) - # GALAXY_URL should be include in the tool params to enable the external application + # GALAXY_URL should be include in the tool params to enable the external application # to send back to the current Galaxy instance GALAXY_URL = incoming.get( 'GALAXY_URL', None ) assert GALAXY_URL is not None, "GALAXY_URL parameter missing in tool config." diff --git a/lib/galaxy/tools/actions/history_imp_exp.py b/lib/galaxy/tools/actions/history_imp_exp.py index 924ae7d2897..fa831a35cd4 100644 --- a/lib/galaxy/tools/actions/history_imp_exp.py +++ b/lib/galaxy/tools/actions/history_imp_exp.py @@ -34,7 +34,7 @@ class ImportHistoryToolAction( ToolAction ): archive_dir = os.path.abspath( tempfile.mkdtemp() ) jiha = trans.app.model.JobImportHistoryArchive( job=job, archive_dir=archive_dir ) trans.sa_session.add( jiha ) - + # # Add parameters to job_parameter table. # diff --git a/lib/galaxy/tools/actions/index_genome.py b/lib/galaxy/tools/actions/index_genome.py index cb317a4cc1e..7d33b335d15 100644 --- a/lib/galaxy/tools/actions/index_genome.py +++ b/lib/galaxy/tools/actions/index_genome.py @@ -46,7 +46,7 @@ class GenomeIndexToolAction( ToolAction ): job_wrapper = GenomeIndexToolWrapper( job ) cmd_line = job_wrapper.setup_job( assoc ) - + # # Add parameters to job_parameter table. # @@ -64,4 +64,4 @@ class GenomeIndexToolAction( ToolAction ): log.info( "Added genome index job to the job queue, id: %s" % str( job.id ) ) return job, odict() - + diff --git a/lib/galaxy/tools/actions/metadata.py b/lib/galaxy/tools/actions/metadata.py index 683a538960e..f450c19a2de 100644 --- a/lib/galaxy/tools/actions/metadata.py +++ b/lib/galaxy/tools/actions/metadata.py @@ -13,15 +13,15 @@ class SetMetadataToolAction( ToolAction ): """ Execute using a web transaction. """ - job, odict = self.execute_via_app( tool, trans.app, trans.get_galaxy_session().id, + job, odict = self.execute_via_app( tool, trans.app, trans.get_galaxy_session().id, trans.history.id, trans.user, incoming, set_output_hid, overwrite, history, job_params ) # FIXME: can remove this when logging in execute_via_app method. trans.log_event( "Added set external metadata job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id ) return job, odict - - def execute_via_app( self, tool, app, session_id, history_id, user=None, - incoming = {}, set_output_hid = False, overwrite = True, + + def execute_via_app( self, tool, app, session_id, history_id, user=None, + incoming = {}, set_output_hid = False, overwrite = True, history=None, job_params=None ): """ Execute using application. @@ -41,7 +41,7 @@ class SetMetadataToolAction( ToolAction ): raise Exception( 'The dataset to set metadata on could not be determined.' ) sa_session = app.model.context - + # Create the job object job = app.model.Job() job.session_id = session_id @@ -61,9 +61,9 @@ class SetMetadataToolAction( ToolAction ): job.set_handler(tool.get_job_handler( job_params )) sa_session.add( job ) sa_session.flush() #ensure job.id is available - + #add parameters to job_parameter table - # Store original dataset state, so we can restore it. A separate table might be better (no chance of 'losing' the original state)? + # Store original dataset state, so we can restore it. A separate table might be better (no chance of 'losing' the original state)? incoming[ '__ORIGINAL_DATASET_STATE__' ] = dataset.state external_metadata_wrapper = JobExternalOutputMetadataWrapper( job ) cmd_line = external_metadata_wrapper.setup_external_metadata( dataset, @@ -90,13 +90,13 @@ class SetMetadataToolAction( ToolAction ): dataset._state = dataset.states.SETTING_METADATA job.state = start_job_state #job inputs have been configured, restore initial job state sa_session.flush() - + # Queue the job for execution app.job_queue.put( job.id, tool.id ) # FIXME: need to add event logging to app and log events there rather than trans. #trans.log_event( "Added set external metadata job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id ) - + #clear e.g. converted files dataset.datatype.before_setting_metadata( dataset ) - + return job, odict() diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index cbf2e27c12b..ff05c32562d 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -18,10 +18,10 @@ class UploadToolAction( ToolAction ): # are in an admin view, and this tool is currently not used there. uploaded_datasets = upload_common.get_uploaded_datasets( trans, '', incoming, precreated_datasets, dataset_upload_inputs, history=history ) upload_common.cleanup_unused_precreated_datasets( precreated_datasets ) - + if not uploaded_datasets: return None, 'No data was entered in the upload form, please go back and choose data to upload.' - + json_file_path = upload_common.create_paramfile( trans, uploaded_datasets ) data_list = [ ud.data for ud in uploaded_datasets ] return upload_common.create_job( trans, incoming, tool, json_file_path, data_list, history=history ) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 7fe8fa6ce3d..cec5e1b109d 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -112,7 +112,7 @@ def __new_history_upload( trans, uploaded_dataset, history=None, state=None ): history = trans.history hda = trans.app.model.HistoryDatasetAssociation( name = uploaded_dataset.name, extension = uploaded_dataset.file_type, - dbkey = uploaded_dataset.dbkey, + dbkey = uploaded_dataset.dbkey, history = history, create_dataset = True, sa_session = trans.sa_session ) diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index c85892a4cb0..51837e9aafe 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -157,30 +157,30 @@ class ToolDataTable( object ): # increment this variable any time a new entry is added, or when the table is totally reloaded # This value has no external meaning, and does not represent an abstract version of the underlying data self._loaded_content_version = 1 - + def _update_version( self ): self._loaded_content_version += 1 return self._loaded_content_version - + def get_empty_field_by_name( self, name ): return self.empty_field_values.get( name, self.empty_field_value ) - + def _add_entry( self, entry, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ): raise NotImplementedError( "Abstract method" ) - + def add_entry( self, entry, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ): self._add_entry( entry, allow_duplicates=allow_duplicates, persist=persist, persist_on_error=persist_on_error, entry_source=entry_source, **kwd ) return self._update_version() - + def add_entries( self, entries, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ): if entries: for entry in entries: self.add_entry( entry, allow_duplicates=allow_duplicates, persist=persist, persist_on_error=persist_on_error, entry_source=entry_source, **kwd ) return self._loaded_content_version - + def is_current_version( self, other_version ): return self._loaded_content_version == other_version - + def merge_tool_data_table( self, other_table, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ): raise NotImplementedError( "Abstract method" ) @@ -212,11 +212,11 @@ class TabularToolDataTable( ToolDataTable ): self.comment_char = config_element.get( 'comment_char', '#' ) # Configure columns self.parse_column_spec( config_element ) - + #store repo info if available: repo_elem = config_element.find( 'tool_shed_repository' ) if repo_elem is not None: - repo_info = dict( tool_shed=repo_elem.find( 'tool_shed' ).text, name=repo_elem.find( 'repository_name' ).text, + repo_info = dict( tool_shed=repo_elem.find( 'tool_shed' ).text, name=repo_elem.find( 'repository_name' ).text, owner=repo_elem.find( 'repository_owner' ).text, installed_changeset_revision=repo_elem.find( 'installed_changeset_revision' ).text ) else: repo_info = None @@ -227,12 +227,12 @@ class TabularToolDataTable( ToolDataTable ): if file_path is None: log.debug( "Encountered a file element (%s) that does not contain a path value when loading tool data table '%s'.", util.xml_to_string( file_element ), self.name ) continue - + #FIXME: splitting on and merging paths from a configuration file when loading is wonky # Data should exist on disk in the state needed, i.e. the xml configuration should # point directly to the desired file to load. Munging of the tool_data_tables_conf.xml.sample # can be done during installing / testing / metadata resetting with the creation of a proper - # tool_data_tables_conf.xml file, containing correct attributes. Allowing a + # tool_data_tables_conf.xml file, containing correct attributes. Allowing a # path.join with a different root should be allowed, but splitting should not be necessary. if tool_data_path and from_shed_config: # Must identify with from_shed_config as well, because the @@ -254,21 +254,21 @@ class TabularToolDataTable( ToolDataTable ): if os.path.exists( corrected_filename ): filename = corrected_filename found = True - + if found: self.data.extend( self.parse_file_fields( open( filename ) ) ) self._update_version() else: self.missing_index_file = filename log.warn( "Cannot find index file '%s' for tool data table '%s'" % ( filename, self.name ) ) - + if filename not in self.filenames or not self.filenames[ filename ][ 'found' ]: - self.filenames[ filename ] = dict( found=found, filename=filename, from_shed_config=from_shed_config, tool_data_path=tool_data_path, + self.filenames[ filename ] = dict( found=found, filename=filename, from_shed_config=from_shed_config, tool_data_path=tool_data_path, config_element=config_element, tool_shed_repository=repo_info ) else: log.debug( "Filename '%s' already exists in filenames (%s), not adding", filename, self.filenames.keys() ) - - + + def merge_tool_data_table( self, other_table, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ): assert self.columns == other_table.columns, "Merging tabular data tables with non matching columns is not allowed: %s:%s != %s:%s" % ( self.name, self.columns, other_table.name, other_table.columns ) #merge filename info @@ -277,14 +277,14 @@ class TabularToolDataTable( ToolDataTable ): self.filenames[ filename ] = info #add data entries and return current data table version return self.add_entries( other_table.data, allow_duplicates=allow_duplicates, persist=persist, persist_on_error=persist_on_error, entry_source=entry_source, **kwd ) - + def handle_found_index_file( self, filename ): self.missing_index_file = None self.data.extend( self.parse_file_fields( open( filename ) ) ) def get_fields( self ): return self.data - + def get_version_fields( self ): return ( self._loaded_content_version, self.data ) @@ -377,7 +377,7 @@ class TabularToolDataTable( ToolDataTable ): rval = fields[ return_col ] break return rval - + def _add_entry( self, entry, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ): #accepts dict or list of columns if isinstance( entry, dict ): @@ -403,7 +403,7 @@ class TabularToolDataTable( ToolDataTable ): log.error( "Attempted to add fields (%s) to data table '%s', but there were not enough fields specified ( %i < %i ).", fields, self.name, len( fields ), self.largest_index + 1 ) is_error = True filename = None - + if persist and ( not is_error or persist_on_error ): if entry_source: #if dict, assume is compatible info dict, otherwise call method @@ -438,7 +438,7 @@ class TabularToolDataTable( ToolDataTable ): data_table_fh.write( '\n' ) data_table_fh.write( "%s\n" % ( self.separator.join( fields ) ) ) return not is_error - + def _replace_field_separators( self, fields, separator=None, replace=None, comment_char=None ): #make sure none of the fields contain separator #make sure separator replace is different from comment_char, @@ -457,6 +457,6 @@ class TabularToolDataTable( ToolDataTable ): else: replace = " " return map( lambda x: x.replace( separator, replace ), fields ) - + # Registry of tool data types by type_key tool_data_table_types = dict( [ ( cls.type_key, cls ) for cls in [ TabularToolDataTable ] ] ) diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index 9d6b2965ecd..2d3b500e8a1 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -95,7 +95,7 @@ class DataManagers( object ): class DataManager( object ): GUID_TYPE = 'data_manager' DEFAULT_VERSION = "0.0.1" - + def __init__( self, data_managers, elem=None, tool_path=None ): self.data_managers = data_managers self.declared_id = None @@ -144,7 +144,7 @@ class DataManager( object ): self.load_tool( os.path.join( tool_path, path ), guid=tool_guid, data_manager_id=self.id, tool_shed_repository_id=tool_shed_repository_id ) self.name = elem.get( 'name', self.tool.name ) self.description = elem.get( 'description', self.tool.description ) - + for data_table_elem in elem.findall( 'data_table' ): data_table_name = data_table_elem.get( "name" ) assert data_table_name is not None, "A name is required for a data table entry" @@ -210,7 +210,7 @@ class DataManager( object ): self.data_managers.app.toolbox.tools_by_id[ tool.id ] = tool self.tool = tool return tool - + def process_result( self, out_data ): data_manager_dicts = {} data_manager_dict = {} @@ -227,7 +227,7 @@ class DataManager( object ): data_manager_dict[ key ] = {} data_manager_dict[ key ].update( value ) data_manager_dict.update( output_dict ) - + data_tables_dict = data_manager_dict.get( 'data_tables', {} ) for data_table_name, data_table_columns in self.data_tables.iteritems(): data_table_values = data_tables_dict.pop( data_table_name, None ) @@ -247,7 +247,7 @@ class DataManager( object ): output_ref_dataset = out_data.get( output_ref, None ) assert output_ref_dataset is not None, "Referenced output was not found." output_ref_values[ data_table_column ] = output_ref_dataset - + if not isinstance( data_table_values, list ): data_table_values = [ data_table_values ] for data_table_row in data_table_values: @@ -257,12 +257,12 @@ class DataManager( object ): moved = self.process_move( data_table_name, name, output_ref_values[ name ].extra_files_path, **data_table_value ) data_table_value[ name ] = self.process_value_translation( data_table_name, name, **data_table_value ) data_table.add_entry( data_table_value, persist=True, entry_source=self ) - + for data_table_name, data_table_values in data_tables_dict.iteritems(): #tool returned extra data table entries, but data table was not declared in data manager #do not add these values, but do provide messages log.warning( 'The data manager "%s" returned an undeclared data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( self.id, data_table_name, data_table_values, data_table_name, self.data_managers.filename ) ) - + def process_move( self, data_table_name, column_name, source_base_path, relative_symlinks=False, **kwd ): if data_table_name in self.move_by_data_table_column and column_name in self.move_by_data_table_column[ data_table_name ]: move_dict = self.move_by_data_table_column[ data_table_name ][ column_name ] @@ -280,7 +280,7 @@ class DataManager( object ): target = fill_template( target, GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd ) if move_dict[ 'target_value' ]: target = os.path.join( target, fill_template( move_dict[ 'target_value' ], GALAXY_DATA_MANAGER_DATA_PATH=self.data_managers.app.config.galaxy_data_manager_data_path, **kwd ) ) - + if move_dict[ 'type' ] == 'file': dirs, filename = os.path.split( target ) try: @@ -291,13 +291,13 @@ class DataManager( object ): #log.debug( 'Error creating directory "%s": %s' % ( dirs, e ) ) #moving a directory and the target already exists, we move the contents instead util.move_merge( source, target ) - + if move_dict.get( 'relativize_symlinks', False ): util.relativize_symlinks( target ) - + return True return False - + def process_value_translation( self, data_table_name, column_name, **kwd ): value = kwd.get( column_name ) if data_table_name in self.value_translation_by_data_table_column and column_name in self.value_translation_by_data_table_column[ data_table_name ]: @@ -307,6 +307,6 @@ class DataManager( object ): else: value = value_translation( value ) return value - + def get_tool_shed_repository_info_dict( self ): return self.tool_shed_repository_info_dict diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index d804584c4b3..c910337b53f 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -30,7 +30,7 @@ class DependencyManager( object ): self.base_paths.append( os.path.abspath( base_path ) ) def find_dep( self, name, version=None, type='package', installed_tool_dependencies=None ): """ - Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a + Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a symbolic link (if found). Returns a triple of: env_script, base_path, real_version """ if version is None: diff --git a/lib/galaxy/tools/deps/tests.py b/lib/galaxy/tools/deps/tests.py index 92bb693ec40..9c3aaa9f6c4 100644 --- a/lib/galaxy/tools/deps/tests.py +++ b/lib/galaxy/tools/deps/tests.py @@ -16,9 +16,9 @@ def test(): # mkdir( base_path ) for name, version, sub in [ ( "dep1", "1.0", "env.sh" ), ( "dep1", "2.0", "bin" ), ( "dep2", "1.0", None ) ]: if sub == "bin": - p = os.path.join( base_path, name, version, "bin" ) + p = os.path.join( base_path, name, version, "bin" ) else: - p = os.path.join( base_path, name, version ) + p = os.path.join( base_path, name, version ) try: makedirs( p ) except: diff --git a/lib/galaxy/tools/exception_handling.py b/lib/galaxy/tools/exception_handling.py index 81479a60bc2..f65b9a09469 100644 --- a/lib/galaxy/tools/exception_handling.py +++ b/lib/galaxy/tools/exception_handling.py @@ -28,6 +28,6 @@ class UCSCOutWrapper( object ): raise UCSCLimitException( next_line.strip() ) else: self.lookahead = next_line - return line + return line def readline(self): return self.next() diff --git a/lib/galaxy/tools/genome_index/__init__.py b/lib/galaxy/tools/genome_index/__init__.py index 4448018e729..80b2287fba8 100644 --- a/lib/galaxy/tools/genome_index/__init__.py +++ b/lib/galaxy/tools/genome_index/__init__.py @@ -34,7 +34,7 @@ def load_genome_index_tools( toolbox ): """ - + # Load index tool. tmp_name = tempfile.NamedTemporaryFile() tmp_name.write( tool_xml_text ) @@ -42,33 +42,33 @@ def load_genome_index_tools( toolbox ): genome_index_tool = toolbox.load_tool( tmp_name.name ) toolbox.tools_by_id[ genome_index_tool.id ] = genome_index_tool log.debug( "Loaded genome index tool: %s", genome_index_tool.id ) - + class GenomeIndexToolWrapper( object ): """ Provides support for performing jobs that index a genome. """ def __init__( self, job_id ): self.locations = dict() self.job_id = job_id - + def setup_job( self, genobj ): - """ Perform setup for job to index a genome and return an archive. Method generates + """ Perform setup for job to index a genome and return an archive. Method generates attribute files, sets the corresponding attributes in the associated database object, and returns a command line for running the job. The command line - includes the command, inputs, and options; it does not include the output + includes the command, inputs, and options; it does not include the output file because it must be set at runtime. """ - + # # Create and return command line for running tool. # scriptpath = os.path.join( os.path.abspath( os.getcwd() ), "lib/galaxy/tools/genome_index/index_genome.py" ) return "python %s %s %s" % ( scriptpath, genobj.indexer, genobj.fasta_path ) - + def postprocessing( self, sa_session, app ): """ Finish the job, move the finished indexes to their final resting place, and update the .loc files where applicable. """ gitd = sa_session.query( model.GenomeIndexToolData ).filter_by( job_id=self.job_id ).first() - indexdirs = dict( bfast='bfast_index', bowtie='bowtie_index', bowtie2='bowtie2_index', + indexdirs = dict( bfast='bfast_index', bowtie='bowtie_index', bowtie2='bowtie2_index', bwa='bwa_index', perm='perm_%s_index', picard='srma_index', sam='sam_index' ) - + if gitd: fp = open( gitd.dataset.get_file_name(), 'r' ) @@ -165,7 +165,7 @@ class GenomeIndexToolWrapper( object ): locfile = self.locations[ 'sam_fa_indexes' ] locdir = os.path.join( destination, line ) location.append( dict( line='\t'.join( [ 'index', dbkey, locdir ] ), file=locfile ) ) - + if destination is not None and os.path.exists( os.path.split( destination )[0] ) and not os.path.exists( destination ): log.debug( 'Moving %s to %s' % ( indexdata, destination ) ) shutil.move( indexdata, destination ) @@ -184,7 +184,7 @@ class GenomeIndexToolWrapper( object ): sa_session.add( deferred ) sa_session.flush() - + def _check_link( self, targetfile, symlink ): target = os.path.relpath( targetfile, os.path.dirname( symlink ) ) filename = os.path.basename( targetfile ) @@ -210,22 +210,22 @@ class GenomeIndexToolWrapper( object ): return else: raise Exception, "Regular file %s exists, is not empty, contents do not match %s." % ( symlink, targetfile ) - + def _hash_file( self, filename ): import hashlib md5 = hashlib.md5() - with open( filename, 'rb' ) as f: + with open( filename, 'rb' ) as f: for chunk in iter( lambda: f.read( 8192 ), '' ): md5.update( chunk ) return md5.digest() - + def _ex_tar( self, directory, filename ): fh = tarfile.open( os.path.join( directory, filename ) ) fh.extractall( path=directory ) fh.close() os.remove( os.path.join( directory, filename ) ) - + def _add_line( self, locfile, newline ): filepath = locfile origlines = [] diff --git a/lib/galaxy/tools/genome_index/index_genome.py b/lib/galaxy/tools/genome_index/index_genome.py index 7ac8aa85015..7c7017a9966 100644 --- a/lib/galaxy/tools/genome_index/index_genome.py +++ b/lib/galaxy/tools/genome_index/index_genome.py @@ -37,7 +37,7 @@ class ManagedIndexer(): if not os.path.exists( self.workingdir ): os.makedirs( self.workingdir ) self.logfile = open( os.path.join( self.workingdir, 'ManagedIndexer.log' ), 'w+' ) - + def run_indexer( self, indexer ): self.fapath = self.fasta self.fafile = os.path.basename( self.fapath ) @@ -59,13 +59,13 @@ class ManagedIndexer(): self._log( 'Indexer %s completed successfully.' % indexer ) self._flush_files() exit(0) - + def _check_link( self ): self._log( 'Checking symlink to %s' % self.fafile ) if not os.path.exists( self.fafile ): self._log( 'Symlink not found, creating' ) os.symlink( os.path.relpath( self.fapath ), self.fafile ) - + def _do_rsync( self, idxpath ): self._log( 'Trying rsync at %s/%s%s' % ( self.rsync_url, self.genome, idxpath ) ) result = subprocess.call( shlex.split( 'rsync %s %s/%s%s .' % ( self.rsync_opts, self.rsync_url, self.genome, idxpath ) ), stderr=self.logfile ) @@ -74,16 +74,16 @@ class ManagedIndexer(): else: self._log( 'Rsync succeeded.' ) return result - + def _flush_files( self ): simplejson.dump( self.locations, self.outfile ) self.outfile.close() self.logfile.close() - + def _log( self, stuff ): timestamp = time.strftime('%Y-%m-%d %H:%M:%S %z') self.logfile.write( "[%s] %s\n" % (timestamp, stuff) ) - + def _bwa( self ): result = self._do_rsync( '/bwa_index/' ) if result == 0: @@ -103,7 +103,7 @@ class ManagedIndexer(): else: self._log( 'BWA (base) exited with code %s' % result ) return False - + def _bwa_cs( self ): if not os.path.exists( os.path.join( self.workingdir, 'cs' ) ): os.makedirs( 'cs' ) @@ -130,8 +130,8 @@ class ManagedIndexer(): temptar.close() shutil.rmtree( 'cs' ) return True - - + + def _bowtie( self ): result = self._do_rsync( '/bowtie_index/' ) if result == 0: @@ -148,7 +148,7 @@ class ManagedIndexer(): else: self._log( 'Bowtie (base) exited with code %s' % result ) return False - + def _bowtie_cs( self ): indexdir = os.path.join( os.getcwd(), 'cs' ) if not ( os.path.exists( indexdir ) ): @@ -168,10 +168,10 @@ class ManagedIndexer(): temptar = tarfile.open( 'cs.tar', 'w' ) temptar.add( 'cs' ) temptar.close() - shutil.rmtree( 'cs' ) + shutil.rmtree( 'cs' ) return True - + def _bowtie2( self ): result = self._do_rsync( '/bowtie2_index/' ) if result == 0: @@ -188,7 +188,7 @@ class ManagedIndexer(): else: self._log( 'Bowtie2 exited with code %s' % result ) return False - + def _twobit( self ): """Index reference files using 2bit for random access. """ @@ -208,7 +208,7 @@ class ManagedIndexer(): else: self._log( 'faToTwoBit exited with code %s' % result ) return False - + def _perm( self ): result = self._do_rsync( '/perm_index/' ) self._check_link() @@ -276,7 +276,7 @@ class ManagedIndexer(): self.locations[ 'nt' ].append( self.fafile ) os.remove( self.fafile ) return True - + def _sam( self ): local_ref = self.fafile local_file = os.path.splitext( self.fafile )[ 0 ] @@ -305,14 +305,14 @@ class WithChDir(): os.chdir( self.working ) def __exit__( self, *args ): os.chdir( self.previous ) - - + + if __name__ == "__main__": # Parse command line. parser = optparse.OptionParser() - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() indexer, infile, outfile, working_dir, rsync_url, tooldata = args - + # Create archive. idxobj = ManagedIndexer( outfile, infile, working_dir, rsync_url, tooldata ) returncode = idxobj.run_indexer( indexer ) diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index cd885ce111c..2639a3fa78d 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -10,7 +10,7 @@ log = logging.getLogger(__name__) def load_history_imp_exp_tools( toolbox ): """ Adds tools for importing/exporting histories to archives. """ - # Use same process as that used in load_external_metadata_tool; see that + # Use same process as that used in load_external_metadata_tool; see that # method for why create tool description files on the fly. tool_xml_text = """ @@ -27,7 +27,7 @@ def load_history_imp_exp_tools( toolbox ): """ - + # Load export tool. tmp_name = tempfile.NamedTemporaryFile() tmp_name.write( tool_xml_text ) @@ -35,15 +35,15 @@ def load_history_imp_exp_tools( toolbox ): history_exp_tool = toolbox.load_tool( tmp_name.name ) toolbox.tools_by_id[ history_exp_tool.id ] = history_exp_tool log.debug( "Loaded history export tool: %s", history_exp_tool.id ) - + # Load import tool. tool_xml = os.path.join( os.getcwd(), "lib/galaxy/tools/imp_exp/imp_history_from_archive.xml" ) history_imp_tool = toolbox.load_tool( tool_xml ) toolbox.tools_by_id[ history_imp_tool.id ] = history_imp_tool log.debug( "Loaded history import tool: %s", history_imp_tool.id ) - + class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations ): - """ + """ Class provides support for performing jobs that import a history from an archive. """ @@ -51,19 +51,19 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations self.app = app self.job_id = job_id self.sa_session = self.app.model.context - + def cleanup_after_job( self ): """ Set history, datasets, and jobs' attributes and clean up archive directory. """ - + # # Helper methods. # - + def file_in_dir( file_path, a_dir ): """ Returns true if file is in directory. """ abs_file_path = os.path.abspath( file_path ) return os.path.split( abs_file_path )[0] == a_dir - + def read_file_contents( file_path ): """ Read contents of a file. """ fp = open( file_path, 'rb' ) @@ -78,24 +78,24 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations pass fp.close() return file_contents - + def get_tag_str( tag, value ): """ Builds a tag string for a tag, value pair. """ if not value: return tag else: return tag + ":" + value - + # # Import history. # - + jiha = self.sa_session.query( model.JobImportHistoryArchive ).filter_by( job_id=self.job_id ).first() if jiha: try: archive_dir = jiha.archive_dir user = jiha.job.user - + # # Create history. # @@ -112,7 +112,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations self.sa_session.add( new_history ) jiha.history = new_history self.sa_session.flush() - + # Add annotation, tags. if user: self.add_item_annotation( self.sa_session, user, new_history, history_attrs[ 'annotation' ] ) @@ -128,7 +128,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations datasets_attrs_file_name = os.path.join( archive_dir, 'datasets_attrs.txt') datasets_attr_str = read_file_contents( datasets_attrs_file_name ) datasets_attrs = from_json_string( datasets_attr_str ) - + # Get counts of how often each dataset file is used; a file can # be linked to multiple dataset objects (HDAs). datasets_usage_counts = {} @@ -138,11 +138,11 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations if ( temp_dataset_file_name not in datasets_usage_counts ): datasets_usage_counts[ temp_dataset_file_name ] = 0 datasets_usage_counts[ temp_dataset_file_name ] += 1 - + # Create datasets. for dataset_attrs in datasets_attrs: metadata = dataset_attrs['metadata'] - + # Create dataset and HDA. hda = model.HistoryDatasetAssociation( name = dataset_attrs['name'].encode( 'utf-8' ), extension = dataset_attrs['extension'], @@ -152,7 +152,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations designation = dataset_attrs['designation'], visible = dataset_attrs['visible'], dbkey = metadata['dbkey'], - metadata = metadata, + metadata = metadata, history = new_history, create_dataset = True, sa_session = self.sa_session ) @@ -165,7 +165,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations #permissions = trans.app.security_agent.history_get_default_permissions( new_history ) #trans.app.security_agent.set_all_dataset_permissions( hda.dataset, permissions ) self.sa_session.flush() - + # Do security check and move/copy dataset data. temp_dataset_file_name = \ os.path.abspath( os.path.join( archive_dir, dataset_attrs['file_name'] ) ) @@ -176,7 +176,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations else: datasets_usage_counts[ temp_dataset_file_name ] -= 1 shutil.copyfile( temp_dataset_file_name, hda.file_name ) - + # Set tags, annotations. if user: self.add_item_annotation( self.sa_session, user, hda, dataset_attrs[ 'annotation' ] ) @@ -189,29 +189,29 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations # Although metadata is set above, need to set metadata to recover BAI for BAMs. if hda.extension == 'bam': - self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute_via_app( - self.app.datatypes_registry.set_external_metadata_tool, self.app, jiha.job.session_id, + self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute_via_app( + self.app.datatypes_registry.set_external_metadata_tool, self.app, jiha.job.session_id, new_history.id, jiha.job.user, incoming={ 'input1': hda }, overwrite=False ) - + # # Create jobs. # - + # Read jobs attributes. jobs_attr_file_name = os.path.join( archive_dir, 'jobs_attrs.txt') jobs_attr_str = read_file_contents( jobs_attr_file_name ) - + # Decode jobs attributes. def as_hda( obj_dct ): - """ Hook to 'decode' an HDA; method uses history and HID to get the HDA represented by + """ Hook to 'decode' an HDA; method uses history and HID to get the HDA represented by the encoded object. This only works because HDAs are created above. """ if obj_dct.get( '__HistoryDatasetAssociation__', False ): return self.sa_session.query( model.HistoryDatasetAssociation ) \ .filter_by( history=new_history, hid=obj_dct['hid'] ).first() return obj_dct jobs_attrs = from_json_string( jobs_attr_str, object_hook=as_hda ) - + # Create each job. for job_attrs in jobs_attrs: imported_job = model.Job() @@ -225,7 +225,7 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations imported_job.imported = True self.sa_session.add( imported_job ) self.sa_session.flush() - + class HistoryDatasetAssociationIDEncoder( simplejson.JSONEncoder ): """ Custom JSONEncoder for a HistoryDatasetAssociation that encodes an HDA as its ID. """ def default( self, obj ): @@ -233,12 +233,12 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations if isinstance( obj, model.HistoryDatasetAssociation ): return obj.id return simplejson.JSONEncoder.default( self, obj ) - + # Set parameters. May be useful to look at metadata.py for creating parameters. # TODO: there may be a better way to set parameters, e.g.: # for name, value in tool.params_to_strings( incoming, trans.app ).iteritems(): # job.add_parameter( name, value ) - # to make this work, we'd need to flesh out the HDA objects. The code below is + # to make this work, we'd need to flesh out the HDA objects. The code below is # relatively similar. for name, value in job_attrs[ 'params' ].items(): # Transform parameter values when necessary. @@ -249,9 +249,9 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations value = input_hda.id #print "added parameter %s-->%s to job %i" % ( name, value, imported_job.id ) imported_job.add_parameter( name, to_json_string( value, cls=HistoryDatasetAssociationIDEncoder ) ) - + # TODO: Connect jobs to input datasets. - + # Connect jobs to output datasets. for output_hid in job_attrs[ 'output_datasets' ]: #print "%s job has output dataset %i" % (imported_job.id, output_hid) @@ -259,36 +259,36 @@ class JobImportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations .filter_by( history=new_history, hid=output_hid ).first() if output_hda: imported_job.add_output_dataset( output_hda.name, output_hda ) - + self.sa_session.flush() # Done importing. new_history.importing = False self.sa_session.flush() - + # Cleanup. if os.path.exists( archive_dir ): - shutil.rmtree( archive_dir ) + shutil.rmtree( archive_dir ) except Exception, e: jiha.job.stderr += "Error cleaning up history import job: %s" % e self.sa_session.flush() class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations ): - """ + """ Class provides support for performing jobs that export a history to an - archive. + archive. """ def __init__( self, job_id ): self.job_id = job_id - + # TODO: should use db_session rather than trans in this method. def setup_job( self, trans, jeha, include_hidden=False, include_deleted=False ): - """ Perform setup for job to export a history into an archive. Method generates + """ Perform setup for job to export a history into an archive. Method generates attribute files for export, sets the corresponding attributes in the jeha object, and returns a command line for running the job. The command line - includes the command, inputs, and options; it does not include the output + includes the command, inputs, and options; it does not include the output file because it must be set at runtime. """ - + # # Helper methods/classes. # @@ -301,7 +301,7 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations tag_user_value = to_unicode( tag.user_value ) tags[ tag_user_tname ] = tag_user_value return tags - + def prepare_metadata( metadata ): """ Prepare metatdata for exporting. """ for name, value in metadata.items(): @@ -310,7 +310,7 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations if isinstance( value, trans.app.model.MetadataFile ): del metadata[ name ] return metadata - + class HistoryDatasetAssociationEncoder( simplejson.JSONEncoder ): """ Custom JSONEncoder for a HistoryDatasetAssociation. """ def default( self, obj ): @@ -338,12 +338,12 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations if isinstance( obj, UnvalidatedValue ): return obj.__str__() return simplejson.JSONEncoder.default( self, obj ) - + # # Create attributes/metadata files for export. - # + # temp_output_dir = tempfile.mkdtemp() - + # Write history attributes to file. history = jeha.history history_attrs = { @@ -362,7 +362,7 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations history_attrs_out.write( to_json_string( history_attrs ) ) history_attrs_out.close() jeha.history_attrs_filename = history_attrs_filename - + # Write datasets' attributes to file. datasets = self.get_history_datasets( trans, history ) included_datasets = [] @@ -396,7 +396,7 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations if not job_hda.creating_job_associations: # No viable HDA found. continue - + # Get the job object. job = None for assoc in job_hda.creating_job_associations: @@ -405,9 +405,9 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations if not job: # No viable job. continue - + jobs_dict[ job.id ] = job - + # Get jobs' attributes. jobs_attrs = [] for id, job in jobs_dict.items(): @@ -415,7 +415,7 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations job_attrs[ 'tool_id' ] = job.tool_id job_attrs[ 'tool_version' ] = job.tool_version job_attrs[ 'state' ] = job.state - + # Get the job's parameters try: params_objects = job.get_param_values( trans.app ) @@ -427,9 +427,9 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations for name, value in params_objects.items(): params_dict[ name ] = value job_attrs[ 'params' ] = params_dict - + # -- Get input, output datasets. -- - + input_datasets = [] for assoc in job.input_datasets: # Optional data inputs will not have a dataset. @@ -438,25 +438,25 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations job_attrs[ 'input_datasets' ] = input_datasets output_datasets = [ assoc.dataset.hid for assoc in job.output_datasets ] job_attrs[ 'output_datasets' ] = output_datasets - + jobs_attrs.append( job_attrs ) - + jobs_attrs_filename = tempfile.NamedTemporaryFile( dir=temp_output_dir ).name jobs_attrs_out = open( jobs_attrs_filename, 'w' ) jobs_attrs_out.write( to_json_string( jobs_attrs, cls=HistoryDatasetAssociationEncoder ) ) jobs_attrs_out.close() jeha.jobs_attrs_filename = jobs_attrs_filename - + # # Create and return command line for running tool. # options = "" if jeha.compressed: options = "-G" - return "python %s %s %s %s %s" % ( + return "python %s %s %s %s %s" % ( os.path.join( os.path.abspath( os.getcwd() ), "lib/galaxy/tools/imp_exp/export_history.py" ), \ options, history_attrs_filename, datasets_attrs_filename, jobs_attrs_filename ) - + def cleanup_after_job( self, db_session ): """ Remove temporary directory and attribute files generated during setup for this job. """ # Get jeha for job. @@ -472,4 +472,4 @@ class JobExportHistoryArchiveWrapper( object, UsesHistoryMixin, UsesAnnotations shutil.rmtree( temp_dir ) except Exception, e: log.debug( 'Error deleting directory containing attribute files (%s): %s' % ( temp_dir, e ) ) - + diff --git a/lib/galaxy/tools/imp_exp/export_history.py b/lib/galaxy/tools/imp_exp/export_history.py index 7e078379a8c..9faba677cc6 100644 --- a/lib/galaxy/tools/imp_exp/export_history.py +++ b/lib/galaxy/tools/imp_exp/export_history.py @@ -24,9 +24,9 @@ def create_archive( history_attrs_file, datasets_attrs_file, jobs_attrs_file, ou if gzip: tarfile_mode += ":gz" try: - + history_archive = tarfile.open( out_file, tarfile_mode ) - + # Read datasets attributes from file. datasets_attr_in = open( datasets_attrs_file, 'rb' ) datasets_attr_str = '' @@ -40,28 +40,28 @@ def create_archive( history_attrs_file, datasets_attrs_file, jobs_attrs_file, ou pass datasets_attr_in.close() datasets_attrs = from_json_string( datasets_attr_str ) - + # Add datasets to archive and update dataset attributes. # TODO: security check to ensure that files added are in Galaxy dataset directory? for dataset_attrs in datasets_attrs: dataset_file_name = dataset_attrs[ 'file_name' ] # Full file name. - dataset_archive_name = os.path.join( 'datasets', + dataset_archive_name = os.path.join( 'datasets', get_dataset_filename( dataset_attrs[ 'name' ], dataset_attrs[ 'extension' ] ) ) history_archive.add( dataset_file_name, arcname=dataset_archive_name ) # Update dataset filename to be archive name. dataset_attrs[ 'file_name' ] = dataset_archive_name - + # Rewrite dataset attributes file. datasets_attrs_out = open( datasets_attrs_file, 'w' ) datasets_attrs_out.write( to_json_string( datasets_attrs ) ) datasets_attrs_out.close() - + # Finish archive. history_archive.add( history_attrs_file, arcname="history_attrs.txt" ) history_archive.add( datasets_attrs_file, arcname="datasets_attrs.txt" ) history_archive.add( jobs_attrs_file, arcname="jobs_attrs.txt" ) history_archive.close() - + # Status. return 'Created history archive.' except Exception, e: @@ -71,10 +71,10 @@ if __name__ == "__main__": # Parse command line. parser = optparse.OptionParser() parser.add_option( '-G', '--gzip', dest='gzip', action="store_true", help='Compress archive using gzip.' ) - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() gzip = bool( options.gzip ) history_attrs, dataset_attrs, job_attrs, out_file = args - + # Create archive. status = create_archive( history_attrs, dataset_attrs, job_attrs, out_file, gzip ) print status \ No newline at end of file diff --git a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py index c1154afb85f..c03053418d1 100644 --- a/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py +++ b/lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py @@ -8,7 +8,7 @@ usage: %prog archive_source dest_dir import sys, optparse, tarfile, tempfile, urllib2, math -# Set max size of archive/file that will be handled to be 100 GB. This is +# Set max size of archive/file that will be handled to be 100 GB. This is # arbitrary and should be adjusted as needed. MAX_SIZE = 100 * math.pow( 2, 30 ) @@ -23,7 +23,7 @@ def url_to_file( url, dest_file ): fp = open( dest_file, 'wb') while True: chunk = url_reader.read( CHUNK ) - if not chunk: + if not chunk: break fp.write( chunk ) total += CHUNK @@ -34,7 +34,7 @@ def url_to_file( url, dest_file ): except Exception, e: print "Exception getting file from URL: %s" % e, sys.stderr return None - + def unpack_archive( archive_file, dest_dir ): """ Unpack a tar and/or gzipped archive into a destination directory. @@ -48,18 +48,18 @@ if __name__ == "__main__": parser = optparse.OptionParser() parser.add_option( '-U', '--url', dest='is_url', action="store_true", help='Source is a URL.' ) parser.add_option( '-F', '--file', dest='is_file', action="store_true", help='Source is a URL.' ) - (options, args) = parser.parse_args() + (options, args) = parser.parse_args() is_url = bool( options.is_url ) is_file = bool( options.is_file ) archive_source, dest_dir = args - + try: # Get archive from URL. if is_url: archive_file = url_to_file( archive_source, tempfile.NamedTemporaryFile( dir=dest_dir ).name ) elif is_file: archive_file = archive_source - + # Unpack archive. unpack_archive( archive_file, dest_dir ) except Exception, e: diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py index 3f91f636cec..0219f67492a 100644 --- a/lib/galaxy/tools/parameters/__init__.py +++ b/lib/galaxy/tools/parameters/__init__.py @@ -12,13 +12,13 @@ def visit_input_values( inputs, input_values, callback, name_prefix="", label_pr parameter `values`, call `callback` for each non-grouping parameter, passing the parameter object, value, a constructed unique name, and a display label. - + If the callback returns a value, it will be replace the old value. - + FIXME: There is redundancy between this and the visit_inputs methods of Repeat and Group. This tracks labels and those do not. It would be nice to unify all the places that recursively visit inputs. - """ + """ for input in inputs.itervalues(): if isinstance( input, Repeat ) or isinstance( input, UploadDataset ): for i, d in enumerate( input_values[ input.name ] ): @@ -42,10 +42,10 @@ def visit_input_values( inputs, input_values, callback, name_prefix="", label_pr def check_param( trans, param, incoming_value, param_values ): """ - Check the value of a single parameter `param`. The value in + Check the value of a single parameter `param`. The value in `incoming_value` is converted from its HTML encoding and validated. - The `param_values` argument contains the processed values of - previous parameters (this may actually be an ExpressionContext + The `param_values` argument contains the processed values of + previous parameters (this may actually be an ExpressionContext when dealing with grouping scenarios). """ value = incoming_value @@ -60,7 +60,7 @@ def check_param( trans, param, incoming_value, param_values ): param.validate( filtered_value, trans.history ) elif value is None and isinstance( param, SelectToolParameter ): # An empty select list or column list - param.validate( value, trans.history ) + param.validate( value, trans.history ) except ValueError, e: error = str( e ) return value, error @@ -70,7 +70,7 @@ def params_to_strings( params, param_values, app ): Convert a dictionary of parameter values to a dictionary of strings suitable for persisting. The `value_to_basic` method of each parameter is called to convert its value to basic types, the result of which - is then json encoded (this allowing complex nested parameters and + is then json encoded (this allowing complex nested parameters and such). """ rval = dict() @@ -79,7 +79,7 @@ def params_to_strings( params, param_values, app ): value = params[ key ].value_to_basic( value, app ) rval[ key ] = str( to_json_string( value ) ) return rval - + def params_from_strings( params, param_values, app, ignore_errors=False ): """ Convert a dictionary of strings as produced by `params_to_strings` @@ -92,16 +92,16 @@ def params_from_strings( params, param_values, app, ignore_errors=False ): value = json_fix( from_json_string( value ) ) if key in params: value = params[key].value_from_basic( value, app, ignore_errors ) - rval[ key ] = value + rval[ key ] = value return rval def params_to_incoming( incoming, inputs, input_values, app, name_prefix="" ): """ Given a tool's parameter definition (`inputs`) and a specific set of parameter `input_values` objects, populate `incoming` with the html values. - + Useful for e.g. the rerun function. - """ + """ for input in inputs.itervalues(): if isinstance( input, Repeat ) or isinstance( input, UploadDataset ): for i, d in enumerate( input_values[ input.name ] ): @@ -116,4 +116,4 @@ def params_to_incoming( incoming, inputs, input_values, app, name_prefix="" ): params_to_incoming( incoming, input.cases[current].inputs, values, app, new_name_prefix ) else: incoming[ name_prefix + input.name ] = input.to_html_value( input_values.get( input.name ), app ) - + diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 60b16b1aac3..cd20bb9a9e4 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -60,18 +60,18 @@ class ToolParameter( object, DictifiableMixin ): def get_html( self, trans=None, value=None, other_values={}): """ - Returns the html widget corresponding to the parameter. + Returns the html widget corresponding to the parameter. Optionally attempt to retain the current value specific by 'value' """ return self.get_html_field( trans, value, other_values ).get_html() - + def from_html( self, value, trans=None, other_values={} ): """ - Convert a value from an HTML POST into the parameters preferred value - format. + Convert a value from an HTML POST into the parameters preferred value + format. """ return value - + def get_initial_value( self, trans, context ): """ Return the starting value of the parameter @@ -95,39 +95,39 @@ class ToolParameter( object, DictifiableMixin ): any encoding) """ return None - + def get_dependencies( self ): """ Return the names of any other parameters this parameter depends on """ return [] - + def filter_value( self, value, trans=None, other_values={} ): """ Parse the value returned by the view into a form usable by the tool OR raise a ValueError. """ return value - + def to_html_value( self, value, app ): """Convert an object value to the value expected from an html post""" return self.to_string( value, app ) - + def to_string( self, value, app ): """Convert a value to a string representation suitable for persisting""" if not isinstance( value, basestring ): value = str( value ) return unicodify( value ) - + def to_python( self, value, app ): """Convert a value created with to_string back to an object representation""" return value - + def value_to_basic( self, value, app ): if isinstance( value, RuntimeValue ): return { "__class__": "RuntimeValue" } return self.to_string( value, app ) - + def value_from_basic( self, value, app, ignore_errors=False ): # HACK: Some things don't deal with unicode well, psycopg problem? if type( value ) == unicode: @@ -143,14 +143,14 @@ class ToolParameter( object, DictifiableMixin ): return value else: return self.to_python( value, app ) - + def value_to_display_text( self, value, app ): """ Convert a value to a text representation suitable for displaying to the user """ return unicodify( value ) - + def to_param_dict_string( self, value, other_values={} ): """Called via __str__ when used in the Cheetah template""" if value is None: @@ -163,7 +163,7 @@ class ToolParameter( object, DictifiableMixin ): else: value = sanitize_param( value ) return value - + def validate( self, value, history=None ): if value=="" and self.optional: return @@ -192,11 +192,11 @@ class ToolParameter( object, DictifiableMixin ): raise ValueError( "Tool parameter '%s' uses an unknown type '%s'" % ( param_name, param_type ) ) else: return parameter_types[param_type]( tool, param ) - + class TextToolParameter( ToolParameter ): """ Parameter that can take on any text value. - + >>> p = TextToolParameter( None, XML( '' ) ) >>> print p.name blah @@ -222,7 +222,7 @@ class TextToolParameter( ToolParameter ): class IntegerToolParameter( TextToolParameter ): """ Parameter that takes an integer value. - + >>> p = IntegerToolParameter( None, XML( '' ) ) >>> print p.name blah @@ -261,15 +261,15 @@ class IntegerToolParameter( TextToolParameter ): raise ValueError( "An integer is required" ) if self.min and self.max: self.validators.append( validation.InRangeValidator( None, self.min, self.max ) ) - + def get_html_field( self, trans=None, value=None, other_values={} ): if isinstance( value, int ): value = str( value ) return super( IntegerToolParameter, self ).get_html_field( trans=trans, value=value, other_values=other_values ) def from_html( self, value, trans=None, other_values={} ): - try: + try: return int( value ) - except: + except: if not value and self.optional: return "" raise ValueError( "An integer is required" ) @@ -295,7 +295,7 @@ class IntegerToolParameter( TextToolParameter ): class FloatToolParameter( TextToolParameter ): """ Parameter that takes a real number value. - + >>> p = FloatToolParameter( None, XML( '' ) ) >>> print p.name blah @@ -339,7 +339,7 @@ class FloatToolParameter( TextToolParameter ): value = str( value ) return super( FloatToolParameter, self ).get_html_field( trans=trans, value=value, other_values=other_values ) def from_html( self, value, trans=None, other_values={} ): - try: + try: return float( value ) except: if not value and self.optional: @@ -366,8 +366,8 @@ class FloatToolParameter( TextToolParameter ): class BooleanToolParameter( ToolParameter ): """ - Parameter that takes one of two values. - + Parameter that takes one of two values. + >>> p = BooleanToolParameter( None, XML( '' ) ) >>> print p.name blah @@ -389,11 +389,11 @@ class BooleanToolParameter( ToolParameter ): self.checked = string_as_bool( elem.get( 'checked' ) ) def get_html_field( self, trans=None, value=None, other_values={} ): checked = self.checked - if value is not None: + if value is not None: checked = form_builder.CheckboxField.is_checked( value ) return form_builder.CheckboxField( self.name, checked, refresh_on_change = self.refresh_on_change ) def from_html( self, value, trans=None, other_values={} ): - return form_builder.CheckboxField.is_checked( value ) + return form_builder.CheckboxField.is_checked( value ) def to_html_value( self, value, app ): if value: return [ 'true', 'true' ] @@ -415,7 +415,7 @@ class BooleanToolParameter( ToolParameter ): class FileToolParameter( ToolParameter ): """ Parameter that takes an uploaded file as a value. - + >>> p = FileToolParameter( None, XML( '' ) ) >>> print p.name blah @@ -476,7 +476,7 @@ class FileToolParameter( ToolParameter ): raise Exception( "FileToolParameter cannot be persisted" ) def get_initial_value( self, trans, context ): return None - + class FTPFileToolParameter( ToolParameter ): """ Parameter that takes a file uploaded via FTP as a value. @@ -518,17 +518,17 @@ class FTPFileToolParameter( ToolParameter ): class HiddenToolParameter( ToolParameter ): """ - Parameter that takes one of two values. - + Parameter that takes one of two values. + FIXME: This seems hacky, parameters should only describe things the user might change. It is used for 'initializing' the UCSC proxy tool - + >>> p = HiddenToolParameter( None, XML( '' ) ) >>> print p.name blah >>> print p.get_html() - """ + """ def __init__( self, tool, elem ): ToolParameter.__init__( self, tool, elem ) self.value = elem.get( 'value' ) @@ -538,14 +538,14 @@ class HiddenToolParameter( ToolParameter ): return self.value def get_label( self ): return None - + ## This is clearly a HACK, parameters should only be used for things the user ## can change, there needs to be a different way to specify this. I'm leaving ## it for now to avoid breaking any tools. class BaseURLToolParameter( ToolParameter ): """ - Returns a parameter the contains its value prepended by the + Returns a parameter the contains its value prepended by the current server base url. Used in all redirects. """ def __init__( self, tool, elem ): @@ -566,8 +566,8 @@ class BaseURLToolParameter( ToolParameter ): class SelectToolParameter( ToolParameter ): """ Parameter that takes on one (or many) or a specific set of values. - - >>> p = SelectToolParameter( None, XML( + + >>> p = SelectToolParameter( None, XML( ... ''' ... ... @@ -592,7 +592,7 @@ class SelectToolParameter( ToolParameter ): >>> print p.filter_value( "y" ) y - >>> p = SelectToolParameter( None, XML( + >>> p = SelectToolParameter( None, XML( ... ''' ... ... @@ -616,8 +616,8 @@ class SelectToolParameter( ToolParameter ): >>> print p.to_param_dict_string( ["y", "z"] ) y,z - - >>> p = SelectToolParameter( None, XML( + + >>> p = SelectToolParameter( None, XML( ... ''' ... ... @@ -682,9 +682,9 @@ class SelectToolParameter( ToolParameter ): elif self.dynamic_options: return set( v for _, v, _ in eval( self.dynamic_options, self.tool.code_namespace, other_values ) ) else: - return self.legal_values + return self.legal_values def get_html_field( self, trans=None, value=None, context={} ): - # Dynamic options are not yet supported in workflow, allow + # Dynamic options are not yet supported in workflow, allow # specifying the value as text for now. if self.need_late_validation( trans, context ): if value is not None: @@ -706,14 +706,14 @@ class SelectToolParameter( ToolParameter ): if isinstance( optval, UnvalidatedValue ): optval = optval.value text = "%s (unvalidated)" % text - if value: + if value: selected = ( optval in value ) field.add_option( text, optval, selected ) return field def from_html( self, value, trans=None, context={} ): if self.need_late_validation( trans, context ): if self.multiple: - # While it is generally allowed that a select value can be '', + # While it is generally allowed that a select value can be '', # we do not allow this to be the case in a dynamically # generated multiple select list being set in workflow building # mode we instead treat '' as 'No option Selected' (None) @@ -731,7 +731,7 @@ class SelectToolParameter( ToolParameter ): if not(self.repeat): assert self.multiple, "Multiple values provided but parameter is not expecting multiple values" rval = [] - for v in value: + for v in value: if v not in legal_values: raise ValueError( "An invalid option was selected, please verify" ) rval.append( v ) @@ -825,7 +825,7 @@ class SelectToolParameter( ToolParameter ): if hasattr( self, 'ref_input' ) and isinstance( dep_value, self.tool.app.model.HistoryDatasetAssociation ) and ( dep_value.is_pending or not dep_value.datatype.matches_any( self.ref_input.formats ) ): return True # Dynamic, but all dependenceis are known and have values - return False + return False def get_initial_value( self, trans, context ): # More working around dynamic options for workflow if self.need_late_validation( trans, context ): @@ -892,20 +892,20 @@ class SelectToolParameter( ToolParameter ): class GenomeBuildParameter( SelectToolParameter ): """ - Select list that sets the last used genome build for the current history + Select list that sets the last used genome build for the current history as "selected". - + >>> # Create a mock transaction with 'hg17' as the current build >>> from galaxy.util.bunch import Bunch >>> trans = Bunch( history=Bunch( genome_build='hg17' ), db_builds=util.dbnames ) - - >>> p = GenomeBuildParameter( None, XML( + + >>> p = GenomeBuildParameter( None, XML( ... ''' ... ... ''' ) ) >>> print p.name blah - + >>> # hg17 should be selected by default >>> print p.get_html( trans ) # doctest: +ELLIPSIS - + >>> # If the user selected something else already, that should be used >>> # instead >>> print p.get_html( trans, value='hg18' ) # doctest: +ELLIPSIS @@ -926,7 +926,7 @@ class GenomeBuildParameter( SelectToolParameter ): ... - + >>> print p.filter_value( "hg17" ) hg17 """ @@ -949,7 +949,7 @@ class ColumnListParameter( SelectToolParameter ): """ Select list that consists of either the total number of columns or only those columns that contain numerical values in the associated DataToolParameter. - + # TODO: we need better testing here, but not sure how to associate a DatatoolParameter with a ColumnListParameter # from a twill perspective... @@ -978,7 +978,7 @@ class ColumnListParameter( SelectToolParameter ): self.ref_input = None self.default_value = elem.get( "default_value", None ) self.is_dynamic = True - self.usecolnames = string_as_bool( elem.get( "use_header_names", False )) + self.usecolnames = string_as_bool( elem.get( "use_header_names", False )) def from_html( self, value, trans=None, context={} ): """ @@ -1012,7 +1012,7 @@ class ColumnListParameter( SelectToolParameter ): def get_column_list( self, trans, other_values ): """ - Generate a select list containing the columns of the associated + Generate a select list containing the columns of the associated dataset (if found). """ column_list = [] @@ -1024,7 +1024,7 @@ class ColumnListParameter( SelectToolParameter ): # Check if a dataset is selected if dataset is None or dataset == '': # NOTE: Both of these values indicate that no dataset is selected. - # However, 'None' indicates that the dataset is optional + # However, 'None' indicates that the dataset is optional # while '' indicates that it is not. Currently column # parameters do not work well with optional datasets return column_list @@ -1057,8 +1057,8 @@ class ColumnListParameter( SelectToolParameter ): cnames = head.rstrip().split('\t') column_list = [('%d' % (i+1),'c%d: %s' % (i+1,x)) for i,x in enumerate(cnames)] if self.numerical: # If numerical was requested, filter columns based on metadata - if len(dataset.metadata.column_types) >= len(cnames): - numerics = [i for i,x in enumerate(dataset.metadata.column_types) if x == 'int' or x == 'float'] + if len(dataset.metadata.column_types) >= len(cnames): + numerics = [i for i,x in enumerate(dataset.metadata.column_types) if x == 'int' or x == 'float'] column_list = [column_list[i] for i in numerics] except: column_list = self.get_column_list( trans, other_values ) @@ -1099,14 +1099,14 @@ class ColumnListParameter( SelectToolParameter ): return True # No late validation return False - + class DrillDownSelectToolParameter( SelectToolParameter ): """ Parameter that takes on one (or many) of a specific set of values. Creating a hierarchical select menu, which allows users to 'drill down' a tree-like set of options. - - >>> p = DrillDownSelectToolParameter( None, XML( + + >>> p = DrillDownSelectToolParameter( None, XML( ... ''' ... ... @@ -1152,7 +1152,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): Option 5
    - >>> p = DrillDownSelectToolParameter( None, XML( + >>> p = DrillDownSelectToolParameter( None, XML( ... ''' ... ... @@ -1237,15 +1237,15 @@ class DrillDownSelectToolParameter( SelectToolParameter ): recurse_option_elems( self.filtered[filter.get( 'data_ref' )][filter.get( 'meta_key' )][filter.get( 'value' )], filter.find( 'options' ).findall( 'option' ) ) elif not self.dynamic_options: recurse_option_elems( self.options, elem.find( 'options' ).findall( 'option' ) ) - + def _get_options_from_code( self, trans=None, value=None, other_values=None ): assert self.dynamic_options, Exception( "dynamic_options was not specifed" ) call_other_values = { '__trans__': trans, '__value__': value } if other_values: call_other_values.update( other_values.dict ) return eval( self.dynamic_options, self.tool.code_namespace, call_other_values ) - - + + def get_options( self, trans=None, value=None, other_values={} ): if self.is_dynamic: if self.dynamic_options: @@ -1263,7 +1263,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): options.extend( meta_dict[check_meta_val] ) return options return self.options - + def get_legal_values( self, trans, other_values ): def recurse_options( legal_values, options ): for option in options: @@ -1272,16 +1272,16 @@ class DrillDownSelectToolParameter( SelectToolParameter ): legal_values = [] recurse_options( legal_values, self.get_options( trans=trans, other_values=other_values ) ) return legal_values - + def get_html( self, trans=None, value=None, other_values={} ): """ - Returns the html widget corresponding to the paramter. + Returns the html widget corresponding to the paramter. Optionally attempt to retain the current value specific by 'value' - """ + """ return self.get_html_field( trans, value, other_values ).get_html() - + def get_html_field( self, trans=None, value=None, other_values={} ): - # Dynamic options are not yet supported in workflow, allow + # Dynamic options are not yet supported in workflow, allow # specifying the value as text for now. if self.need_late_validation( trans, other_values ): if value is not None: @@ -1296,7 +1296,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): else: return form_builder.TextField( self.name, value=(value or "") ) return form_builder.DrillDownField( self.name, self.multiple, self.display, self.refresh_on_change, self.get_options( trans, value, other_values ), value, refresh_on_change_values = self.refresh_on_change_values ) - + def from_html( self, value, trans=None, other_values={} ): if self.need_late_validation( trans, other_values ): if self.multiple: @@ -1315,7 +1315,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): if val not in self.get_legal_values( trans, other_values ): raise ValueError( "An invalid option was selected, please verify" ) rval.append( val ) return rval - + def to_param_dict_string( self, value, other_values={} ): def get_options_list( value ): def get_base_option( value, options ): @@ -1334,7 +1334,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): rval = [] recurse_option( rval, get_base_option( value, self.get_options( other_values = other_values ) ) ) return rval or [value] - + if value is None: return "None" rval = [] if self.hierarchy == "exact": @@ -1353,7 +1353,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): else: rval = sanitize_param( rval ) return rval - + def get_initial_value( self, trans, context ): def recurse_options( initial_values, options ): for option in options: @@ -1376,7 +1376,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): rval = get_option_display( value, option['options'] ) if rval: return rval return None #not found - + if isinstance( value, UnvalidatedValue ): suffix = "\n(value not yet validated)" value = value.value @@ -1401,7 +1401,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ): for val in value: rval.append( get_option_display( val, self.options ) or val ) return "\n".join( map( str, rval ) ) + suffix - + def get_dependencies( self ): """ Get the *names* of the other params this param depends on. @@ -1417,7 +1417,7 @@ class DataToolParameter( ToolParameter ): """ Parameter that takes on one (or many) or a specific set of values. - TODO: There should be an alternate display that allows single selects to be + TODO: There should be an alternate display that allows single selects to be displayed as radio buttons and multiple selects as a set of checkboxes TODO: The following must be fixed to test correctly for the new security_check tag in the DataToolParameter ( the last test below is broken ) @@ -1459,7 +1459,7 @@ class DataToolParameter( ToolParameter ): self.options_filter_attribute = None else: self.options = dynamic_options.DynamicOptions( options, self ) - + #HACK to get around current hardcoded limitation of when a set of dynamic options is defined for a DataToolParameter #it always causes available datasets to be filtered by dbkey #this behavior needs to be entirely reworked (in a backwards compatible manner) @@ -1548,7 +1548,7 @@ class DataToolParameter( ToolParameter ): def get_initial_value_from_history_prevent_repeats( self, trans, context, already_used ): """ NOTE: This is wasteful since dynamic options and dataset collection - happens twice (here and when generating HTML). + happens twice (here and when generating HTML). """ # Can't look at history in workflow mode. Tool shed has no histories. if trans is None or trans.workflow_building_mode or trans.webapp.name == 'tool_shed': @@ -1644,7 +1644,7 @@ class DataToolParameter( ToolParameter ): return str( value ) def to_python( self, value, app ): - # Both of these values indicate that no dataset is selected. However, 'None' + # Both of these values indicate that no dataset is selected. However, 'None' # indicates that the dataset is optional, while '' indicates that it is not. if value is None or value == '' or value == 'None': return value @@ -1657,7 +1657,7 @@ class DataToolParameter( ToolParameter ): def to_param_dict_string( self, value, other_values={} ): if value is None: return "None" return value.file_name - + def value_to_display_text( self, value, app ): if value and not isinstance( value, list ): value = [ value ] @@ -1719,20 +1719,20 @@ class DataToolParameter( ToolParameter ): class HiddenDataToolParameter( HiddenToolParameter, DataToolParameter ): """ - Hidden parameter that behaves as a DataToolParameter. As with all hidden + Hidden parameter that behaves as a DataToolParameter. As with all hidden parameters, this is a HACK. """ def __init__( self, tool, elem ): DataToolParameter.__init__( self, tool, elem ) self.value = "None" self.type = "hidden_data" - + def get_initial_value( self, trans, context ): return None - + def get_html_field( self, trans=None, value=None, other_values={} ): return form_builder.HiddenField( self.name, self.value ) - + class LibraryDatasetToolParameter( ToolParameter ): """ @@ -1741,10 +1741,10 @@ class LibraryDatasetToolParameter( ToolParameter ): def __init__( self, tool, elem ): ToolParameter.__init__( self, tool, elem ) - + def get_html_field( self, trans=None, value=None, other_values={} ): return form_builder.LibraryField( self.name, value=value, trans=trans ) - + def get_initial_value( self, trans, context ): return None @@ -1775,9 +1775,9 @@ class LibraryDatasetToolParameter( ToolParameter ): # class RawToolParameter( ToolParameter ): # """ # Completely nondescript parameter, HTML representation is provided as text -# contents. -# -# >>> p = RawToolParameter( None, XML( +# contents. +# +# >>> p = RawToolParameter( None, XML( # ... ''' # ... # ... Some random stuff]]> @@ -1795,15 +1795,15 @@ class LibraryDatasetToolParameter( ToolParameter ): # context = dict( self.__dict__ ) # context.update( dict( prefix=prefix ) ) # return self.template.substitute( context ) - + # class HistoryIDParameter( ToolParameter ): # """ -# Parameter that takes a name value, makes history.id available. -# +# Parameter that takes a name value, makes history.id available. +# # FIXME: This is a hack (esp. if hidden params are a hack) but in order to # have the history accessable at the job level, it is necessary # I also probably wrote this docstring test thing wrong. -# +# # >>> from galaxy.model import History # >>> from galaxy.util.bunch import Bunch # >>> hist = History( id=1 ) @@ -1812,7 +1812,7 @@ class LibraryDatasetToolParameter( ToolParameter ): # blah # >>> html_string = '' % hist.id # >>> assert p.get_html( trans=Bunch( history=hist ) ) == html_string -# """ +# """ # def __init__( self, tool, elem ): # ToolParameter.__init__( self, tool, elem ) # def get_html( self, trans, value=None, other_values={} ): @@ -1844,7 +1844,7 @@ class UnvalidatedValue( object ): self.value = value def __str__( self ): return str( self.value ) - + class RuntimeValue( object ): """ Wrapper to note a value that is not yet set, but will be required at diff --git a/lib/galaxy/tools/parameters/dynamic_options.py b/lib/galaxy/tools/parameters/dynamic_options.py index fbd9ffcb627..8bec122265b 100644 --- a/lib/galaxy/tools/parameters/dynamic_options.py +++ b/lib/galaxy/tools/parameters/dynamic_options.py @@ -33,9 +33,9 @@ class Filter( object ): class StaticValueFilter( Filter ): """ Filters a list of options on a column by a static value. - + Type: static_value - + Required Attributes: value: static value to compare to column: column in options to compare with @@ -67,12 +67,12 @@ class StaticValueFilter( Filter ): class DataMetaFilter( Filter ): """ Filters a list of options on a column by a dataset metadata value. - + Type: data_meta - + When no 'from' source has been specified in the tag, this will populate the options list with (meta_value, meta_value, False). Otherwise, options which do not match the metadata value in the column are discarded. - + Required Attributes: - ref: Name of input dataset @@ -121,7 +121,7 @@ class DataMetaFilter( Filter ): meta_value = ref.metadata.get( self.key, None ) if meta_value is None: #assert meta_value is not None, "Required metadata value '%s' not found in referenced dataset" % self.key return [ ( disp_name, basic.UnvalidatedValue( optval ), selected ) for disp_name, optval, selected in options ] - + if self.column is not None: rval = [] for fields in options: @@ -145,9 +145,9 @@ class DataMetaFilter( Filter ): class ParamValueFilter( Filter ): """ Filters a list of options on a column by the value of another input. - + Type: param_value - + Required Attributes: - ref: Name of input value @@ -193,9 +193,9 @@ class ParamValueFilter( Filter ): class UniqueValueFilter( Filter ): """ Filters a list of options to be unique by a column value. - + Type: unique_value - + Required Attributes: column: column in options to compare with """ @@ -218,9 +218,9 @@ class UniqueValueFilter( Filter ): class MultipleSplitterFilter( Filter ): """ Turns a single line of options into multiple lines, by splitting a column and creating a line for each item. - + Type: multiple_splitter - + Required Attributes: column: column in options to compare with Optional Attributes: @@ -239,7 +239,7 @@ class MultipleSplitterFilter( Filter ): for field in fields[column].split( self.separator ): rval.append( fields[0:column] + [field] + fields[column+1:] ) return rval - + class AttributeValueSplitterFilter( Filter ): """ Filters a list of attribute-value pairs to be unique attribute names. @@ -277,9 +277,9 @@ class AttributeValueSplitterFilter( Filter ): class AdditionalValueFilter( Filter ): """ Adds a single static value to an options list. - + Type: add_value - + Required Attributes: value: value to appear in select list Optional Attributes: @@ -312,9 +312,9 @@ class AdditionalValueFilter( Filter ): class RemoveValueFilter( Filter ): """ Removes a value from an options list. - + Type: remove_value - + Required Attributes:: value: value to remove from select list @@ -363,9 +363,9 @@ class RemoveValueFilter( Filter ): class SortByColumnFilter( Filter ): """ Sorts an options list by a column - + Type: sort_by - + Required Attributes: column: column to sort by """ @@ -417,7 +417,7 @@ class DynamicOptions( object ): self.has_dataset_dependencies = False self.validators = [] self.converter_safe = True - + # Parse the tag self.separator = elem.get( 'separator', '\t' ) self.line_startswith = elem.get( 'startswith', None ) @@ -466,18 +466,18 @@ class DynamicOptions( object ): elif from_parameter is not None: transform_lines = elem.get( 'transform_lines', None ) self.file_fields = list( load_from_parameter( from_parameter, transform_lines ) ) - + # Load filters for filter_elem in elem.findall( 'filter' ): self.filters.append( Filter.from_element( self, filter_elem ) ) - + # Load Validators for validator in elem.findall( 'validator' ): self.validators.append( validation.Validator.from_element( self.tool_param, validator ) ) - + if self.dataset_ref_name: tool_param.data_ref = self.dataset_ref_name - + def parse_column_definitions( self, elem ): for column_elem in elem.findall( 'column' ): name = column_elem.get( 'name', None ) @@ -491,7 +491,7 @@ class DynamicOptions( object ): assert 'value' in self.columns, "Required 'value' column missing from column def" if 'name' not in self.columns: self.columns['name'] = self.columns['value'] - + def parse_file_fields( self, reader ): rval = [] field_count = None @@ -514,7 +514,7 @@ class DynamicOptions( object ): ( field_count, len( fields ), name, self.separator, line ) ) rval.append( fields ) return rval - + def get_dependency_names( self ): """ Return the names of parameters these options depend on -- both data @@ -528,7 +528,7 @@ class DynamicOptions( object ): if depend: rval.append( depend ) return rval - + def get_fields( self, trans, other_values ): if self.dataset_ref_name: dataset = other_values.get( self.dataset_ref_name, None ) @@ -540,7 +540,7 @@ class DynamicOptions( object ): if os.path.getsize( path ) < 1048576: options = self.parse_file_fields( open( path ) ) else: - # Pass just the first megabyte to parse_file_fields. + # Pass just the first megabyte to parse_file_fields. import StringIO log.warn( "Attempting to load options from large file, reading just first megabyte" ) contents = open( path, 'r' ).read( 1048576 ) @@ -552,7 +552,7 @@ class DynamicOptions( object ): for filter in self.filters: options = filter.filter_options( options, trans, other_values ) return options - + def get_fields_by_value( self, value, trans, other_values ): """ Return a list of fields with column 'value' matching provided value. @@ -563,7 +563,7 @@ class DynamicOptions( object ): if fields[ val_index ] == value: rval.append( fields ) return rval - + def get_field_by_name_for_value( self, field_name, value, trans, other_values ): """ Get contents of field by name for specified value. @@ -580,7 +580,7 @@ class DynamicOptions( object ): for fields in self.get_fields_by_value( val, trans, other_values ): rval.append( fields[ field_index ] ) return rval - + def get_options( self, trans, other_values ): rval = [] if self.file_fields is not None or self.tool_data_table is not None or self.dataset_ref_name is not None: @@ -591,7 +591,7 @@ class DynamicOptions( object ): for filter in self.filters: rval = filter.filter_options( rval, trans, other_values ) return rval - + def column_spec_to_index( self, column_spec ): """ Convert a column specification (as read from the config file), to an diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index 694eef587c9..521cc834ae9 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -23,25 +23,25 @@ class Group( object, DictifiableMixin ): def __init__( self ): self.name = None - + @property def visible( self ): return True - + def value_to_basic( self, value, app ): """ Convert value to a (possibly nested) representation using only basic types (dict, list, tuple, str, unicode, int, long, float, bool, None) """ return value - + def value_from_basic( self, value, app, ignore_errors=False ): """ Convert a basic representation as produced by `value_to_basic` back into the preferred value form. """ return value - + def get_initial_value( self, trans, context ): """ Return the initial state/value for this group @@ -52,7 +52,7 @@ class Group( object, DictifiableMixin ): # TODO: need to dictify conditions. group_dict = super( Group, self ).dictify( view=view, value_mapper=value_mapper ) return group_dict - + class Repeat( Group ): type = "repeat" def __init__( self ): @@ -92,7 +92,7 @@ class Repeat( Group ): if ignore_errors and input.name not in d: # If we do not have a value, and are ignoring errors, we simply # do nothing. There will be no value for the parameter in the - # conditional's values dictionary. + # conditional's values dictionary. pass else: rval_dict[ input.name ] = input.value_from_basic( d[input.name], app, ignore_errors ) @@ -100,7 +100,7 @@ class Repeat( Group ): except Exception, e: if not ignore_errors: raise e - return rval + return rval def visit_inputs( self, prefix, value, callback ): for i, d in enumerate( value ): for input in self.inputs.itervalues(): @@ -131,7 +131,7 @@ class UploadDataset( Group ): def get_composite_dataset_name( self, context ): #FIXME: HACK #Special case of using 'base_name' metadata for use as Dataset name needs to be done in a General Fashion, as defined within a particular Datatype. - + #We get two different types of contexts here, one straight from submitted parameters, the other after being parsed into tool inputs dataset_name = context.get('files_metadata|base_name', None ) if dataset_name is None: @@ -193,7 +193,7 @@ class UploadDataset( Group ): else: rval_dict[ input.name ] = input.value_from_basic( d[input.name], app, ignore_errors ) rval.append( rval_dict ) - return rval + return rval def visit_inputs( self, prefix, value, callback ): for i, d in enumerate( value ): for input in self.inputs.itervalues(): @@ -473,7 +473,7 @@ class Conditional( Group ): if ignore_errors and input.name not in value: # If we do not have a value, and are ignoring errors, we simply # do nothing. There will be no value for the parameter in the - # conditional's values dictionary. + # conditional's values dictionary. pass else: rval[ input.name ] = input.value_from_basic( value[ input.name ], app, ignore_errors ) @@ -490,7 +490,7 @@ class Conditional( Group ): else: input.visit_inputs( prefix, value[input.name], callback ) def get_initial_value( self, trans, context ): - # State for a conditional is a plain dictionary. + # State for a conditional is a plain dictionary. rval = {} # Get the default value for the 'test element' and use it # to determine the current case @@ -505,7 +505,7 @@ class Conditional( Group ): for child_input in self.cases[current_case].inputs.itervalues(): rval[ child_input.name ] = child_input.get_initial_value( trans, child_context ) return rval - + class ConditionalWhen( object ): def __init__( self ): self.value = None diff --git a/lib/galaxy/tools/parameters/input_translation.py b/lib/galaxy/tools/parameters/input_translation.py index 47e61c16005..6c4b9051679 100644 --- a/lib/galaxy/tools/parameters/input_translation.py +++ b/lib/galaxy/tools/parameters/input_translation.py @@ -1,5 +1,5 @@ """ -Tool Input Translation. +Tool Input Translation. """ import logging @@ -11,10 +11,10 @@ class ToolInputTranslator( object ): """ Handles Tool input translation. This is used for data source tools - + >>> from galaxy.util import Params >>> from elementtree.ElementTree import XML - >>> translator = ToolInputTranslator.from_element( XML( + >>> translator = ToolInputTranslator.from_element( XML( ... ''' ... ... @@ -57,7 +57,7 @@ class ToolInputTranslator( object ): missing = req_param.get( "missing" ) value_trans = {} append_param = None - + value_trans_elem = req_param.find( 'value_translation' ) if value_trans_elem: for value_elem in value_trans_elem.findall( 'value' ): @@ -65,10 +65,10 @@ class ToolInputTranslator( object ): galaxy_value = value_elem.get( "galaxy_value" ) if None not in [ remote_value, galaxy_value ]: value_trans[ remote_value ] = galaxy_value - + append_param_elem = req_param.find( "append_param" ) if append_param_elem: - separator = append_param_elem.get( 'separator', ',' ) + separator = append_param_elem.get( 'separator', ',' ) first_separator = append_param_elem.get( 'first_separator', None ) join_str = append_param_elem.get( 'join', '=' ) append_dict = {} @@ -78,11 +78,11 @@ class ToolInputTranslator( object ): if None not in [ value_name, value_missing ]: append_dict[ value_name ] = value_missing append_param = Bunch( separator = separator, first_separator = first_separator, join_str = join_str, append_dict = append_dict ) - + rval.param_trans_dict[ remote_name ] = Bunch( galaxy_name = galaxy_name, missing = missing, value_trans = value_trans, append_param = append_param ) - + return rval - + def __init__( self ): self.param_trans_dict = {} diff --git a/lib/galaxy/tools/parameters/output.py b/lib/galaxy/tools/parameters/output.py index 00179bc90c5..9ebc7efc1df 100644 --- a/lib/galaxy/tools/parameters/output.py +++ b/lib/galaxy/tools/parameters/output.py @@ -65,7 +65,7 @@ class ValueToolOutputActionConditionalWhen( ToolOutputActionConditionalWhen ): def is_case( self, output_dataset, other_values ): ref = self.get_ref( output_dataset, other_values ) return bool( str( ref ) == self.value ) - + class DatatypeIsInstanceToolOutputActionConditionalWhen( ToolOutputActionConditionalWhen ): tag = "when datatype_isinstance" def __init__( self, parent, config_elem, value ): @@ -248,7 +248,7 @@ class FormatToolOutputAction( ToolOutputAction ): def __init__( self, parent, elem ): super( FormatToolOutputAction, self ).__init__( parent, elem ) self.default = elem.get( 'default', None ) - + def apply_action( self, output_dataset, other_values ): value = self.option.get_value( other_values ) if value is None and self.default is not None: @@ -431,7 +431,7 @@ class MetadataValueFilter( ToolOutputActionOptionFilter ): value = str( getattr( ref.metadata, self.name ) ) rval = [] for fields in options: - if self.keep == ( self.compare( fields[self.column], value ) ): + if self.keep == ( self.compare( fields[self.column], value ) ): rval.append( fields ) return rval @@ -452,7 +452,7 @@ class BooleanFilter( ToolOutputActionOptionFilter ): value = self.cast( value ) except: value = False #unable to cast or access value; treat as false - if self.keep == bool( value ): + if self.keep == bool( value ): rval.append( fields ) return rval @@ -480,7 +480,7 @@ for action_type in [ MetadataToolOutputAction, FormatToolOutputAction ]: option_types = {} for option_type in [ NullToolOutputActionOption, FromFileToolOutputActionOption, FromParamToolOutputActionOption, FromDataTableOutputActionOption ]: option_types[ option_type.tag ] = option_type - + filter_types = {} for filter_type in [ ParamValueToolOutputActionOptionFilter, InsertColumnToolOutputActionOptionFilter, MultipleSplitterFilter, ColumnStripFilter, MetadataValueFilter, BooleanFilter, StringFunctionFilter, ColumnReplaceFilter ]: filter_types[ filter_type.tag ] = filter_type @@ -524,5 +524,5 @@ def compare_endswith( value1, value2 ): def compare_re_search( value1, value2 ): #checks pattern=value2 in value1 return bool( re.search( value2, value1 ) ) - + compare_types = { 'eq':compare_eq, 'neq':compare_neq, 'gt':compare_gt, 'gte':compare_gte, 'lt':compare_lt, 'lte':compare_lte, 'in':compare_in, 'startswith':compare_startswith, 'endswith':compare_endswith, "re_search":compare_re_search } diff --git a/lib/galaxy/tools/parameters/sanitize.py b/lib/galaxy/tools/parameters/sanitize.py index 50420f93faa..8122abf547e 100644 --- a/lib/galaxy/tools/parameters/sanitize.py +++ b/lib/galaxy/tools/parameters/sanitize.py @@ -1,5 +1,5 @@ """ -Tool Parameter specific sanitizing. +Tool Parameter specific sanitizing. """ import logging @@ -11,9 +11,9 @@ log = logging.getLogger( __name__ ) class ToolParameterSanitizer( object ): """ Handles tool parameter specific sanitizing. - + >>> from elementtree.ElementTree import XML - >>> sanitizer = ToolParameterSanitizer.from_element( XML( + >>> sanitizer = ToolParameterSanitizer.from_element( XML( ... ''' ... ... @@ -22,7 +22,7 @@ class ToolParameterSanitizer( object ): >>> sanitizer.sanitize_param( ''.join( sorted( [ c for c in string.printable ] ) ) ) == ''.join( sorted( [ c for c in string.letters ] ) ) True >>> slash = chr( 92 ) - >>> sanitizer = ToolParameterSanitizer.from_element( XML( + >>> sanitizer = ToolParameterSanitizer.from_element( XML( ... ''' ... ... @@ -40,11 +40,11 @@ class ToolParameterSanitizer( object ): >>> [ c for c in sanitizer.sanitize_param( text ) ] == [ slash, slash, slash, '"', '$', 'r', 'm', '&', '#', '!' ] True """ - + VALID_PRESET = { 'default':( string.letters + string.digits +" -=_.()/+*^,:?!" ), 'none':'' } MAPPING_PRESET = { 'default':galaxy.util.mapped_chars, 'none':{} } DEFAULT_INVALID_CHAR = 'X' - + #class methods @classmethod def from_element( cls, elem ): @@ -85,9 +85,9 @@ class ToolParameterSanitizer( object ): if map_source is not None and map_key in rval._mapped_chars: del rval._mapped_chars[ map_key ] else: - log.debug( 'Invalid action tag in mapping: %s' % action_elem.tag ) + log.debug( 'Invalid action tag in mapping: %s' % action_elem.tag ) return rval - + @classmethod def get_valid_by_name( cls, name ): rval = [] @@ -105,7 +105,7 @@ class ToolParameterSanitizer( object ): log.debug( 'Invalid preset name specified: %s' % split_name ) rval.extend( [ val for val in value if val not in rval ] ) return rval - + @classmethod def get_mapping_by_name( cls, name ): rval = {} @@ -117,20 +117,20 @@ class ToolParameterSanitizer( object ): log.debug( 'Invalid preset name specified: %s' % split_name ) return rval #end class methods - + def __init__( self ): self._valid_chars = [] #List of valid characters self._mapped_chars = {} #Replace a char with a any number of characters self._invalid_char = self.DEFAULT_INVALID_CHAR #Replace invalid characters with this character self.sanitize = True #Simply pass back the passed in value - + def restore_text( self, text ): """Restores sanitized text""" if self.sanitize: for key, value in self._mapped_chars.iteritems(): text = text.replace( value, key ) return text - + def restore_param( self, value ): if self.sanitize: if isinstance( value, basestring ): @@ -140,7 +140,7 @@ class ToolParameterSanitizer( object ): else: raise Exception, 'Unknown parameter type (%s:%s)' % ( type( value ), value ) return value - + def sanitize_text( self, text ): """Restricts the characters that are allowed in a text""" if not self.sanitize: @@ -154,7 +154,7 @@ class ToolParameterSanitizer( object ): else: rval.append( self._invalid_char ) return ''.join( rval ) - + def sanitize_param( self, value ): """Clean incoming parameters (strings or lists)""" if not self.sanitize: diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index 85acd5b1cd4..5b6f78169b3 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -23,11 +23,11 @@ class Validator( object ): return validator_types[type].from_element( param, elem ) def validate( self, value, history=None ): raise TypeError( "Abstract Method" ) - + class RegexValidator( Validator ): """ Validator that evaluates a regular expression - + >>> from galaxy.tools.parameters import ToolParameter >>> p = ToolParameter.build( None, XML( ''' ... @@ -47,16 +47,16 @@ class RegexValidator( Validator ): def __init__( self, message, expression ): self.message = message # Compile later. RE objects used to not be thread safe. Not sure about - # the sre module. - self.expression = expression + # the sre module. + self.expression = expression def validate( self, value, history=None ): if re.match( self.expression, value ) is None: raise ValueError( self.message ) - + class ExpressionValidator( Validator ): """ Validator that evaluates a python expression using the value - + >>> from galaxy.tools.parameters import ToolParameter >>> p = ToolParameter.build( None, XML( ''' ... @@ -84,11 +84,11 @@ class ExpressionValidator( Validator ): if self.substitute_value_in_message: message = message % value raise ValueError( message ) - + class InRangeValidator( Validator ): """ Validator that ensures a number is in a specific range - + >>> from galaxy.tools.parameters import ToolParameter >>> p = ToolParameter.build( None, XML( ''' ... @@ -108,15 +108,15 @@ class InRangeValidator( Validator ): return cls( elem.get( 'message', None ), elem.get( 'min', '-inf' ), elem.get( 'max', '+inf' ) ) def __init__( self, message, range_min, range_max ): self.min = float( range_min ) - self.max = float( range_max ) + self.max = float( range_max ) # Remove unneeded 0s and decimal from floats to make message pretty. self_min_str = str( self.min ).rstrip( '0' ).rstrip( '.' ) self_max_str = str( self.max ).rstrip( '0' ).rstrip( '.' ) self.message = message or "Value must be between %s and %s" % ( self_min_str, self_max_str ) def validate( self, value, history=None ): if not( self.min <= float( value ) <= self.max ): - raise ValueError( self.message ) - + raise ValueError( self.message ) + class LengthValidator( Validator ): """ Validator that ensures the length of the provided string (value) is in a specific range @@ -143,7 +143,7 @@ class LengthValidator( Validator ): return cls( elem.get( 'message', None ), elem.get( 'min', None ), elem.get( 'max', None ) ) def __init__( self, message, length_min, length_max ): self.message = message - if length_min is not None: + if length_min is not None: length_min = int( length_min ) if length_max is not None: length_max = int( length_max ) @@ -293,7 +293,7 @@ class MetadataInDataTableColumnValidator( Validator ): if line_startswith: line_startswith = line_startswith.strip() return cls( tool_data_table, metadata_name, metadata_column, message, line_startswith ) - + def __init__( self, tool_data_table, metadata_name, metadata_column, message="Value for metadata not found.", line_startswith=None ): self.metadata_name = metadata_name self.message = message @@ -304,14 +304,14 @@ class MetadataInDataTableColumnValidator( Validator ): metadata_column = tool_data_table.columns[ metadata_column ] self._metadata_column = metadata_column self._load_values() - + def _load_values( self ): self._data_table_content_version, data_fields = self._tool_data_table.get_version_fields() self.valid_values = [] for fields in data_fields: if self._metadata_column < len( fields ): self.valid_values.append( fields[ self._metadata_column ] ) - + def validate( self, value, history = None ): if not value: return if hasattr( value, "metadata" ): @@ -334,7 +334,7 @@ validator_types = dict( expression=ExpressionValidator, dataset_metadata_in_file=MetadataInFileColumnValidator, dataset_metadata_in_data_table=MetadataInDataTableColumnValidator, dataset_ok_validator=DatasetOkValidator ) - + def get_suite(): """Get unittest suite for this module""" import doctest, sys diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index bd1f242e8d7..acf98537acf 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -14,14 +14,14 @@ class ToolBoxSearch( object ): Support searching tools in a toolbox. This implementation uses the "whoosh" search library. """ - + def __init__( self, toolbox ): """ - Create a searcher for `toolbox`. + Create a searcher for `toolbox`. """ self.toolbox = toolbox self.build_index() - + def build_index( self ): self.storage = RamStorage() self.index = self.storage.create_index( schema ) @@ -30,7 +30,7 @@ class ToolBoxSearch( object ): for id, tool in self.toolbox.tools_by_id.iteritems(): writer.add_document( id=id, title=to_unicode(tool.name), description=to_unicode(tool.description), help=to_unicode(tool.help) ) writer.commit() - + def search( self, query, return_attribute='id' ): # Change field boosts for searcher to place more weight on title, description than help. searcher = self.index.searcher( \ diff --git a/lib/galaxy/tools/test.py b/lib/galaxy/tools/test.py index e4a3001cdc6..ef9c3b55b33 100644 --- a/lib/galaxy/tools/test.py +++ b/lib/galaxy/tools/test.py @@ -11,8 +11,8 @@ log = logging.getLogger( __name__ ) class ToolTestBuilder( object ): """ - Encapsulates information about a tool test, and allows creation of a - dynamic TestCase class (the unittest framework is very class oriented, + Encapsulates information about a tool test, and allows creation of a + dynamic TestCase class (the unittest framework is very class oriented, doing dynamic tests in this way allows better integration) """ def __init__( self, tool, name, maxseconds ): @@ -36,7 +36,7 @@ class ToolTestBuilder( object ): break if not found_parameter: raise ValueError( "Unable to determine parameter type of test input '%s'. " - "Ensure that the parameter exists and that any container groups are defined first." + "Ensure that the parameter exists and that any container groups are defined first." % name ) elif isinstance( self.tool.inputs[name], basic.DataToolParameter ): value = self.__add_uploaded_dataset( name, value, extra, self.tool.inputs[name] ) @@ -80,7 +80,7 @@ class ToolTestBuilder( object ): # is something like "input2" and the expanded page display is something like "queries_0|input2". # The problem is that the only param name on the page is "input2", and adding more test input params # with the same name ( "input2" ) is not yet supported in our test code ( the last one added is the only - # one used ). + # one used ). if name in grouping_value.inputs: if isinstance( grouping_value.inputs[name], basic.DataToolParameter ): return True, self.__add_uploaded_dataset( name, value, extra, grouping_value.inputs[name] ) diff --git a/lib/galaxy/tools/util/galaxyops/__init__.py b/lib/galaxy/tools/util/galaxyops/__init__.py index db1cec4af8a..b8c4b36d0b0 100644 --- a/lib/galaxy/tools/util/galaxyops/__init__.py +++ b/lib/galaxy/tools/util/galaxyops/__init__.py @@ -8,7 +8,7 @@ def warn( msg ): # don't need both a warn and a fail... print >> sys.stderr, msg sys.exit( 1 ) - + def fail( msg ): print >> sys.stderr, msg sys.exit( 1 ) diff --git a/lib/galaxy/tools/util/hyphy_util.py b/lib/galaxy/tools/util/hyphy_util.py index 7164feeaf2b..c420e116473 100644 --- a/lib/galaxy/tools/util/hyphy_util.py +++ b/lib/galaxy/tools/util/hyphy_util.py @@ -1,1163 +1,1163 @@ -#Dan Blankenberg -#Contains file contents and helper methods for HYPHY configurations -import tempfile, os - -def get_filled_temp_filename(contents): - fh = tempfile.NamedTemporaryFile('w') - filename = fh.name - fh.close() - fh = open(filename, 'w') - fh.write(contents) - fh.close() - return filename - -NJ_tree_shared_ibf = """ -COUNT_GAPS_IN_FREQUENCIES = 0; -methodIndex = 1; - -/*-----------------------------------------------------------------------------------------------------------------------------------------*/ - -function InferTreeTopology(verbFlag) -{ - distanceMatrix = {ds.species,ds.species}; - - MESSAGE_LOGGING = 0; - ExecuteAFile (HYPHY_BASE_DIRECTORY+"TemplateBatchFiles"+DIRECTORY_SEPARATOR+"chooseDistanceFormula.def"); - InitializeDistances (0); - - for (i = 0; i methodIndex)>=ds.species; - - treeNodes = {2*(ds.species+1),3}; - cladesInfo = {ds.species-1,2}; - - for (i=Rows(treeNodes)-1; i>=0; i=i-1) - { - treeNodes[i][0] = njm[i][0]; - treeNodes[i][1] = njm[i][1]; - treeNodes[i][2] = njm[i][2]; - } - - for (i=Rows(cladesInfo)-1; i>=0; i=i-1) - { - cladesInfo[i][0] = njm[i][3]; - cladesInfo[i][1] = njm[i][4]; - } - - njm = 0; - } - } - return 1.0; -} - -/*-----------------------------------------------------------------------------------------------------------------------------------------*/ - -function TreeMatrix2TreeString (doLengths) -{ - treeString = ""; - p = 0; - k = 0; - m = treeNodes[0][1]; - n = treeNodes[0][0]; - treeString*(Rows(treeNodes)*25); - - while (m) - { - if (m>p) - { - if (p) - { - treeString*","; - } - for (j=p;j.5) - { - nodeName = ":"+treeNodes[k][2]; - treeString*nodeName; - } - k=k+1; - p=m; - n=treeNodes[k][0]; - m=treeNodes[k][1]; - } - - for (j=m;j methodIndex)>=ds.species; + + treeNodes = {2*(ds.species+1),3}; + cladesInfo = {ds.species-1,2}; + + for (i=Rows(treeNodes)-1; i>=0; i=i-1) + { + treeNodes[i][0] = njm[i][0]; + treeNodes[i][1] = njm[i][1]; + treeNodes[i][2] = njm[i][2]; + } + + for (i=Rows(cladesInfo)-1; i>=0; i=i-1) + { + cladesInfo[i][0] = njm[i][3]; + cladesInfo[i][1] = njm[i][4]; + } + + njm = 0; + } + } + return 1.0; +} + +/*-----------------------------------------------------------------------------------------------------------------------------------------*/ + +function TreeMatrix2TreeString (doLengths) +{ + treeString = ""; + p = 0; + k = 0; + m = treeNodes[0][1]; + n = treeNodes[0][0]; + treeString*(Rows(treeNodes)*25); + + while (m) + { + if (m>p) + { + if (p) + { + treeString*","; + } + for (j=p;j.5) + { + nodeName = ":"+treeNodes[k][2]; + treeString*nodeName; + } + k=k+1; + p=m; + n=treeNodes[k][0]; + m=treeNodes[k][1]; + } + + for (j=m;j>> lines = [ line for line in file_iter(__file__) ] - >>> len(lines) != 0 + >>> len(lines) != 0 True """ for line in file(fname): @@ -122,7 +122,7 @@ def file_reader( fp, chunk_size=CHUNK_SIZE ): def unique_id(KEY_SIZE=128): """ Generates an unique id - + >>> ids = [ unique_id() for i in range(1000) ] >>> len(set(ids)) 1000 @@ -166,7 +166,7 @@ def xml_element_to_dict( elem ): rval[ elem.tag ] = {} else: rval[ elem.tag ] = None - + sub_elems = list( elem ) if sub_elems: sub_elem_dict = dict() @@ -183,14 +183,14 @@ def xml_element_to_dict( elem ): if elem.attrib: for key, value in elem.attrib.iteritems(): rval[ elem.tag ][ "@%s" % key ] = value - + if elem.text: text = elem.text.strip() if text and sub_elems or elem.attrib: rval[ elem.tag ][ '#text' ] = text else: rval[ elem.tag ] = text - + return rval @@ -294,8 +294,8 @@ def pretty_print_json(json_data, is_json_string=False): valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") # characters that are allowed but need to be escaped -mapped_chars = { '>' :'__gt__', - '<' :'__lt__', +mapped_chars = { '>' :'__gt__', + '<' :'__lt__', "'" :'__sq__', '"' :'__dq__', '[' :'__ob__', @@ -371,13 +371,13 @@ def sanitize_for_filename( text, default=None ): class Params( object ): """ - Stores and 'sanitizes' parameters. Alphanumeric characters and the + Stores and 'sanitizes' parameters. Alphanumeric characters and the non-alphanumeric ones that are deemed safe are let to pass through (see L{valid_chars}). - Some non-safe characters are escaped to safe forms for example C{>} becomes C{__lt__} + Some non-safe characters are escaped to safe forms for example C{>} becomes C{__lt__} (see L{mapped_chars}). All other characters are replaced with C{X}. - + Operates on string or list values only (HTTP parameters). - + >>> values = { 'status':'on', 'symbols':[ 'alpha', '<>', '$rm&#!' ] } >>> par = Params(values) >>> par.status @@ -391,14 +391,14 @@ class Params( object ): >>> par.flatten() # flattening to a list [('status', 'on'), ('symbols', 'alpha'), ('symbols', '__lt____gt__'), ('symbols', 'XrmX__pd__!')] """ - + # is NEVER_SANITIZE required now that sanitizing for tool parameters can be controlled on a per parameter basis and occurs via InputValueWrappers? NEVER_SANITIZE = ['file_data', 'url_paste', 'URL', 'filesystem_paths'] - + def __init__( self, params, sanitize=True ): if sanitize: for key, value in params.items(): - if key not in self.NEVER_SANITIZE and True not in [ key.endswith( "|%s" % nonsanitize_parameter ) for nonsanitize_parameter in self.NEVER_SANITIZE ]: #sanitize check both ungrouped and grouped parameters by name. Anything relying on NEVER_SANITIZE should be changed to not require this and NEVER_SANITIZE should be removed. + if key not in self.NEVER_SANITIZE and True not in [ key.endswith( "|%s" % nonsanitize_parameter ) for nonsanitize_parameter in self.NEVER_SANITIZE ]: #sanitize check both ungrouped and grouped parameters by name. Anything relying on NEVER_SANITIZE should be changed to not require this and NEVER_SANITIZE should be removed. self.__dict__[ key ] = sanitize_param( value ) else: self.__dict__[ key ] = value @@ -420,11 +420,11 @@ class Params( object ): def __getattr__(self, name): """This is here to ensure that we get None for non existing parameters""" - return None - + return None + def get(self, key, default): return self.__dict__.get(key, default) - + def __str__(self): return '%s' % self.__dict__ @@ -444,7 +444,7 @@ def rst_to_html( s ): def write( self, str ): if len( str ) > 0 and not str.isspace(): log.warn( str ) - return docutils.core.publish_string(s, + return docutils.core.publish_string(s, writer=docutils.writers.html4css1.Writer(), settings_overrides={"embed_stylesheet": False, "template": os.path.join(os.path.dirname(__file__), "docutils_template.txt"), "warning_stream": FakeStream()}) @@ -464,7 +464,7 @@ def xml_text(root, name=None): return text.strip() # No luck, return empty string return '' - + # asbool implementation pulled from PasteDeploy truthy = frozenset(['true', 'yes', 'on', 'y', 't', '1']) falsy = frozenset(['false', 'no', 'off', 'n', 'f', '0']) @@ -549,10 +549,10 @@ def unicodify( value, encoding=DEFAULT_ENCODING, error='replace', default=None ) def object_to_string( obj ): return binascii.hexlify( pickle.dumps( obj, 2 ) ) - + def string_to_object( s ): return pickle.loads( binascii.unhexlify( s ) ) - + def get_ucsc_by_build(build): sites = [] for site in ucsc_build_sites: @@ -871,7 +871,7 @@ def move_merge( source, target ): for name in os.listdir( source ): move_merge( os.path.join( source, name ), os.path.join( target, name ) ) else: - return shutil.move( source, target ) + return shutil.move( source, target ) galaxy_root_path = os.path.join(__path__[0], "..","..","..") diff --git a/lib/galaxy/util/backports/__init__.py b/lib/galaxy/util/backports/__init__.py index ad702b1b0f2..c53fb7e1a81 100644 --- a/lib/galaxy/util/backports/__init__.py +++ b/lib/galaxy/util/backports/__init__.py @@ -1,3 +1,3 @@ """ Modules for providing backward compatibility with future versions of Python -""" \ No newline at end of file +""" \ No newline at end of file diff --git a/lib/galaxy/util/backports/importlib/__init__.py b/lib/galaxy/util/backports/importlib/__init__.py index c7655084c5f..53a807affae 100644 --- a/lib/galaxy/util/backports/importlib/__init__.py +++ b/lib/galaxy/util/backports/importlib/__init__.py @@ -37,6 +37,6 @@ def import_module(name, package=None): __import__(name) return sys.modules[name] - ## Note: this was copied from + ## Note: this was copied from ## http://svn.python.org/projects/python/trunk/Lib/importlib/__init__.py ## on 24 September 2012 \ No newline at end of file diff --git a/lib/galaxy/util/bunch.py b/lib/galaxy/util/bunch.py index 7d482a3673e..b7dc05ea19f 100644 --- a/lib/galaxy/util/bunch.py +++ b/lib/galaxy/util/bunch.py @@ -2,7 +2,7 @@ class Bunch( object ): """ http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52308 - Often we want to just collect a bunch of stuff together, naming each item of + Often we want to just collect a bunch of stuff together, naming each item of the bunch; a dictionary's OK for that, but a small do-nothing class is even handier, and prettier to use. """ def __init__(self, **kwds): diff --git a/lib/galaxy/util/expressions.py b/lib/galaxy/util/expressions.py index 88452602b16..cce9583c873 100644 --- a/lib/galaxy/util/expressions.py +++ b/lib/galaxy/util/expressions.py @@ -1,7 +1,7 @@ """ Expression evaluation support. -For the moment this depends on python's eval. In the future it should be +For the moment this depends on python's eval. In the future it should be replaced with a "safe" parser. """ diff --git a/lib/galaxy/util/heartbeat.py b/lib/galaxy/util/heartbeat.py index 0e6877edcc6..feb5899d398 100644 --- a/lib/galaxy/util/heartbeat.py +++ b/lib/galaxy/util/heartbeat.py @@ -8,20 +8,20 @@ try: pkg_resources.require( "threadframe" ) except: - + import sys print >> sys.stderr, "No threadframe module, Heartbeat not available" Heartbeat = None - + else: - + import threading import threadframe import time import traceback import os import sys - + def get_current_thread_object_dict(): """ Get a dictionary of all 'Thread' objects created via the threading @@ -29,14 +29,14 @@ else: have a thread objects, only the main thread and any created via the 'threading' module. Threads created via the low level 'thread' module will not be in the returned dictionary. - + HACK: This mucks with the internals of the threading module since that module does not expose any way to match 'Thread' objects with intepreter thread identifiers (though it should). """ rval = dict() # Acquire the lock and then union the contents of 'active' and 'limbo' - # threads into the return value. + # threads into the return value. threading._active_limbo_lock.acquire() rval.update( threading._active ) rval.update( threading._limbo ) @@ -96,12 +96,12 @@ else: # Cleanup self.file.close() self.file_nonsleeping.close() - + def shutdown( self ): self.should_stop = True self.wait_event.set() self.join() - + def thread_is_sleeping ( self, last_stack_frame ): """ Returns True if the given stack-frame represents a known @@ -147,7 +147,7 @@ else: return ( relative_filename, _line, _funcname, _text ) # no "/lib/galaxy" code found, return the innermost frame return stack_frames[-1] - + def print_nonsleeping( self, threads_object_dict ): print >> self.file_nonsleeping, "Non-Sleeping threads at %s:" % time.asctime() print >> self.file_nonsleeping diff --git a/lib/galaxy/util/inflection.py b/lib/galaxy/util/inflection.py index 67342b05593..69ab3f5d011 100644 --- a/lib/galaxy/util/inflection.py +++ b/lib/galaxy/util/inflection.py @@ -10,10 +10,10 @@ import re class Base: '''Locale inflectors must inherit from this base class inorder to provide the basic Inflector functionality''' - + def cond_plural(self, number_of_records, word) : '''Returns the plural form of a word if first parameter is greater than 1''' - + if number_of_records != 1: return self.pluralize(word) else : @@ -26,7 +26,7 @@ class Base: "welcome_page" or "welcome page" to this "Welcome Page". If second parameter is set to 'first' it will only capitalize the first character of the title.''' - + if(uppercase == 'first'): return self.humanize(self.underscore(word)).capitalize() else : @@ -39,18 +39,18 @@ class Base: will remove non alphanumeric character from the word, so "who's online" will be converted to "WhoSOnline"''' return ''.join(w[0].upper() + w[1:] for w in re.sub('[^A-Z^a-z^0-9^:]+', ' ', word).split(' ')) - + def underscore(self, word) : ''' Converts a word "into_it_s_underscored_version" Convert any "CamelCased" or "ordinary Word" into an "underscored_word". This can be really useful for creating friendly URLs.''' - + return re.sub('[^A-Z^a-z^0-9^\/]+','_', \ re.sub('([a-z\d])([A-Z])','\\1_\\2', \ re.sub('([A-Z]+)([A-Z][a-z])','\\1_\\2', re.sub('::', '/',word)))).lower() - - + + def humanize(self, word, uppercase = '') : '''Returns a human-readable string from word Returns a human-readable string from word, by replacing @@ -58,13 +58,13 @@ class Base: character by default. If you need to uppercase all the words you just have to pass 'all' as a second parameter.''' - + if(uppercase == 'first'): return re.sub('_id$', '', word).replace('_',' ').capitalize() else : return re.sub('_id$', '', word).replace('_',' ').title() - - + + def variablize(self, word) : '''Same as camelize but first char is lowercased Converts a word like "send_email" to "sendEmail". It @@ -72,19 +72,19 @@ class Base: "who's online" will be converted to "whoSOnline"''' word = self.camelize(word) return word[0].lower()+word[1:] - + def tableize(self, class_name) : ''' Converts a class name to its table name according to rails naming conventions. Example. Converts "Person" to "people" ''' return self.pluralize(self.underscore(class_name)) - - + + def classify(self, table_name) : '''Converts a table name to its class name according to rails naming conventions. Example: Converts "people" to "Person" ''' return self.camelize(self.singularize(table_name)) - - + + def ordinalize(self, number) : '''Converts number to its ordinal English form. This method converts 13 to 13th, 2 to 2nd ...''' @@ -97,40 +97,40 @@ class Base: tail = 'nd' elif number % 10 == 3 : tail = 'rd' - + return str(number)+tail - - + + def unaccent(self, text) : - '''Transforms a string to its unaccented version. + '''Transforms a string to its unaccented version. This might be useful for generating "friendly" URLs''' find = u'\u00C0\u00C1\u00C2\u00C3\u00C4\u00C5\u00C6\u00C7\u00C8\u00C9\u00CA\u00CB\u00CC\u00CD\u00CE\u00CF\u00D0\u00D1\u00D2\u00D3\u00D4\u00D5\u00D6\u00D8\u00D9\u00DA\u00DB\u00DC\u00DD\u00DE\u00DF\u00E0\u00E1\u00E2\u00E3\u00E4\u00E5\u00E6\u00E7\u00E8\u00E9\u00EA\u00EB\u00EC\u00ED\u00EE\u00EF\u00F0\u00F1\u00F2\u00F3\u00F4\u00F5\u00F6\u00F8\u00F9\u00FA\u00FB\u00FC\u00FD\u00FE\u00FF' replace = u'AAAAAAACEEEEIIIIDNOOOOOOUUUUYTsaaaaaaaceeeeiiiienoooooouuuuyty' return self.string_replace(text, find, replace) - + def string_replace (self, word, find, replace) : '''This function returns a copy of word, translating all occurrences of each character in find to the corresponding character in replace''' for k in range(0,len(find)) : word = re.sub(find[k], replace[k], word) - + return word - + def urlize(self, text) : '''Transform a string its unaccented and underscored version ready to be inserted in friendly URLs''' return re.sub('^_|_$','',self.underscore(self.unaccent(text))) - - + + def demodulize(self, module_name) : return self.humanize(self.underscore(re.sub('^.*::','',module_name))) - + def modulize(self, module_description) : return self.camelize(self.singularize(module_description)) - + def foreignKey(self, class_name, separate_class_name_and_id_with_underscore = 1) : - ''' Returns class_name in underscored form, with "_id" tacked on at the end. + ''' Returns class_name in underscored form, with "_id" tacked on at the end. This is for use in dealing with the database.''' if separate_class_name_and_id_with_underscore : tail = '_id' @@ -272,23 +272,23 @@ class English (Base): class Inflector: """ Inflector for pluralizing and singularizing nouns. - + It provides methods for helping on creating programs based on naming conventions like on Ruby on Rails. """ - + def __init__( self, Inflector = English ) : assert callable(Inflector), "Inflector should be a callable obj" self.Inflector = apply(Inflector); - + def pluralize(self, word) : '''Pluralizes nouns.''' return self.Inflector.pluralize(word) - + def singularize(self, word) : '''Singularizes nouns.''' return self.Inflector.singularize(word) - + def cond_plural(self, number_of_records, word) : '''Returns the plural form of a word if first parameter is greater than 1''' return self.Inflector.cond_plural(number_of_records, word) @@ -307,14 +307,14 @@ class Inflector: will remove non alphanumeric character from the word, so "who's online" will be converted to "WhoSOnline"''' return self.Inflector.camelize(word) - + def underscore(self, word) : ''' Converts a word "into_it_s_underscored_version" Convert any "CamelCased" or "ordinary Word" into an "underscored_word". This can be really useful for creating friendly URLs.''' return self.Inflector.underscore(word) - + def humanize(self, word, uppercase = '') : '''Returns a human-readable string from word Returns a human-readable string from word, by replacing @@ -323,53 +323,53 @@ class Inflector: If you need to uppercase all the words you just have to pass 'all' as a second parameter.''' return self.Inflector.humanize(word, uppercase) - - + + def variablize(self, word) : '''Same as camelize but first char is lowercased Converts a word like "send_email" to "sendEmail". It will remove non alphanumeric character from the word, so "who's online" will be converted to "whoSOnline"''' return self.Inflector.variablize(word) - + def tableize(self, class_name) : ''' Converts a class name to its table name according to rails naming conventions. Example. Converts "Person" to "people" ''' return self.Inflector.tableize(class_name) - + def classify(self, table_name) : '''Converts a table name to its class name according to rails naming conventions. Example: Converts "people" to "Person" ''' return self.Inflector.classify(table_name) - + def ordinalize(self, number) : '''Converts number to its ordinal form. This method converts 13 to 13th, 2 to 2nd ...''' return self.Inflector.ordinalize(number) - - + + def unaccent(self, text) : - '''Transforms a string to its unaccented version. + '''Transforms a string to its unaccented version. This might be useful for generating "friendly" URLs''' return self.Inflector.unaccent(text) - + def urlize(self, text) : '''Transform a string its unaccented and underscored version ready to be inserted in friendly URLs''' return self.Inflector.urlize(text) - - + + def demodulize(self, module_name) : return self.Inflector.demodulize(module_name) - + def modulize(self, module_description) : return self.Inflector.modulize(module_description) - + def foreignKey(self, class_name, separate_class_name_and_id_with_underscore = 1) : - ''' Returns class_name in underscored form, with "_id" tacked on at the end. + ''' Returns class_name in underscored form, with "_id" tacked on at the end. This is for use in dealing with the database.''' return self.Inflector.foreignKey(class_name, separate_class_name_and_id_with_underscore) - + # Copyright (c) 2006 Bermi Ferrer Martinez # Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/lib/galaxy/util/lrucache.py b/lib/galaxy/util/lrucache.py index ec78b6608a1..6b7c4556dfe 100644 --- a/lib/galaxy/util/lrucache.py +++ b/lib/galaxy/util/lrucache.py @@ -10,7 +10,7 @@ class LRUCache: ''' Clears/initiates storage variables''' self.key_ary = [] self.obj_cache = {} - + def __init__(self, num_elements): self.num_elements = num_elements self.clear() @@ -25,7 +25,7 @@ class LRUCache: self.key_ary.remove(key) self.key_ary.append(key) return self.obj_cache[key] - + def __setitem__(self, key, value): ''' Sets a new value to a key ''' if key not in self.obj_cache: @@ -38,7 +38,7 @@ class LRUCache: if __name__ == "__main__": import unittest - + class TestLRUCache(unittest.TestCase): def test_lru(self): lru = LRUCache(2) @@ -48,14 +48,14 @@ if __name__ == "__main__": self.assertEqual( lru[1], None ) self.assertEqual( lru[2], 2 ) self.assertEqual( lru[3], 3 ) - + self.assertEqual( lru.__setitem__("hello", "world"), "world") self.assertEqual( lru[2], None ) - + lru.clear() self.assertEqual( lru["hello"], None ) self.assertEqual( lru[3], None ) - + # Test if recently used item is kept lru[0] = 0 lru[1] = 1 @@ -66,7 +66,6 @@ if __name__ == "__main__": self.assertEqual( lru[0], 0 ) self.assertEqual( lru[1], None ) self.assertEqual( lru[2], 2 ) - + unittest.main() - - \ No newline at end of file + diff --git a/lib/galaxy/util/memdump.py b/lib/galaxy/util/memdump.py index 524d98d9534..25558ca4d47 100644 --- a/lib/galaxy/util/memdump.py +++ b/lib/galaxy/util/memdump.py @@ -8,15 +8,15 @@ try: pkg_resources.require( "guppy" ) except: - + import sys print >> sys.stderr, "No guppy module, Memdump not available" Memdump = None - + else: - + import os, sys, signal, time, guppy - + class Memdump( object ): def __init__( self, signum=signal.SIGUSR1, fname="memdump.log" ): self.fname = fname diff --git a/lib/galaxy/util/odict.py b/lib/galaxy/util/odict.py index 005996d3684..08041bf9fb6 100644 --- a/lib/galaxy/util/odict.py +++ b/lib/galaxy/util/odict.py @@ -10,7 +10,7 @@ class odict(UserDict): This dictionary class extends UserDict to record the order in which items are added. Calling keys(), values(), items(), etc. will return results in this - order. + order. """ def __init__( self, dict = None ): self._keys = [] @@ -70,12 +70,12 @@ class odict(UserDict): def iteritems( self ): for key in self._keys: - yield key, self.get( key ) + yield key, self.get( key ) def __iter__( self ): - for key in self._keys: + for key in self._keys: yield key - + def reverse( self ): self._keys.reverse() diff --git a/lib/galaxy/util/pastescript/loadwsgi.py b/lib/galaxy/util/pastescript/loadwsgi.py index a5eefea47cd..19ad1d3eea0 100644 --- a/lib/galaxy/util/pastescript/loadwsgi.py +++ b/lib/galaxy/util/pastescript/loadwsgi.py @@ -771,7 +771,7 @@ class FuncLoader(_Loader): """ Loader that supports specifying functions inside modules, without using eggs at all. Configuration should be in the format: use = call:my.module.path:function_name - + Dot notation is supported in both the module and function name, e.g.: use = call:my.module.path:object.method """ diff --git a/lib/galaxy/util/pastescript/serve.py b/lib/galaxy/util/pastescript/serve.py index c2e5a800b58..1fc07759ee2 100644 --- a/lib/galaxy/util/pastescript/serve.py +++ b/lib/galaxy/util/pastescript/serve.py @@ -11,9 +11,9 @@ # http://www.mems-exchange.org/software/qp/ # From lib/site.py -# Galaxy originally used PasteScript and PasteDeploy for application -# loading, to maintain compatibility we've internalized some of that -# code here, stripping out uneeded functionality. +# Galaxy originally used PasteScript and PasteDeploy for application +# loading, to maintain compatibility we've internalized some of that +# code here, stripping out uneeded functionality. # All top level imports from each package moved here and organized import ConfigParser @@ -167,7 +167,7 @@ class Command(object): def run(self, args): self.parse_args(args) - + # Setup defaults: for name, default in [('verbose', 0), ('quiet', 0), @@ -243,7 +243,7 @@ class Command(object): ######################################## ## Utility methods ######################################## - + def pad(self, s, length, dir='left'): if len(s) >= length: return s @@ -260,7 +260,7 @@ class Command(object): overwrite=False): """ Create a standard ``OptionParser`` instance. - + Typically used like:: class MyCommand(Command): @@ -484,7 +484,7 @@ class ServeCommand(Command): _monitor_environ_key = 'PASTE_MONITOR_SHOULD_RUN' possible_subcommands = ('start', 'stop', 'restart', 'status') - + def command(self): if self.options.stop_daemon: return self.stop_daemon() @@ -639,7 +639,7 @@ class ServeCommand(Command): self.logging_file_config(log_fn) server = loadserver(server_spec, name=server_name, relative_to=base, global_conf=vars) - + app = loadapp( app_spec, name=app_name, relative_to=base, global_conf=vars) if self.verbose > 0: @@ -850,7 +850,7 @@ class ServeCommand(Command): os.initgroups(user, gid) else: os.setgroups([e.gr_gid for e in grp.getgrall() - if user in e.gr_mem] + [gid]) + if user in e.gr_mem] + [gid]) if gid: os.setgid(gid) if uid: diff --git a/lib/galaxy/util/sanitize_html.py b/lib/galaxy/util/sanitize_html.py index 07720bf89aa..aecd859c61e 100644 --- a/lib/galaxy/util/sanitize_html.py +++ b/lib/galaxy/util/sanitize_html.py @@ -50,13 +50,13 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): bare_ampersand = re.compile("&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)") elements_no_end_tag = ['area', 'base', 'basefont', 'br', 'col', 'frame', 'hr', 'img', 'input', 'isindex', 'link', 'meta', 'param'] - + def __init__(self, encoding, type): self.encoding = encoding self.type = type ## if _debug: sys.stderr.write('entering BaseHTMLProcessor, encoding=%s\n' % self.encoding) sgmllib.SGMLParser.__init__(self) - + def reset(self): self.pieces = [] sgmllib.SGMLParser.reset(self) @@ -78,7 +78,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): def feed(self, data): data = re.compile(r'', self._shorttag_replace, data) # bug [ 1399464 ] Bad regexp for _shorttag_replace - data = re.sub(r'<([^<>\s]+?)\s*/>', self._shorttag_replace, data) + data = re.sub(r'<([^<>\s]+?)\s*/>', self._shorttag_replace, data) data = data.replace(''', "'") data = data.replace('"', '"') if self.encoding and type(data) == type(u''): @@ -141,7 +141,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): self.pieces.append('&#%s;' % hex(ord(_cp1252[value]))[1:]) else: self.pieces.append('&#%(ref)s;' % locals()) - + def handle_entityref(self, ref): # called for each entity reference, e.g. for '©', ref will be 'copy' # Reconstruct the original entity reference. @@ -156,12 +156,12 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): # Store the original text verbatim. ## if _debug: sys.stderr.write('_BaseHTMLProcessor, handle_text, text=%s\n' % text) self.pieces.append(text) - + def handle_comment(self, text): # called for each HTML comment, e.g. # Reconstruct the original comment. self.pieces.append('' % locals()) - + def handle_pi(self, text): # called for each processing instruction, e.g. # Reconstruct original processing instruction. @@ -173,7 +173,7 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser): # "http://www.w3.org/TR/html4/loose.dtd"> # Reconstruct original DOCTYPE self.pieces.append('' % locals()) - + _new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match def _scan_name(self, i, declstartpos): rawdata = self.rawdata @@ -283,7 +283,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): # svgtiny - foreignObject + linearGradient + radialGradient + stop svg_elements = ['a', 'animate', 'animateColor', 'animateMotion', 'animateTransform', 'circle', 'defs', 'desc', 'ellipse', 'foreignObject', - 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', + 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', 'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect', 'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use'] @@ -329,7 +329,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): self.unacceptablestack = 0 self.mathmlOK = 0 self.svgOK = 0 - + def unknown_starttag(self, tag, attrs): acceptable_attributes = self.acceptable_attributes keymap = {} @@ -383,7 +383,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): ## clean_value = self.sanitize_style(value) ## if clean_value: clean_attrs.append((key,clean_value)) _BaseHTMLProcessor.unknown_starttag(self, tag, clean_attrs) - + def unknown_endtag(self, tag): if not tag in self.acceptable_elements: if tag in self.unacceptable_elements_with_end_tag: diff --git a/lib/galaxy/util/topsort.py b/lib/galaxy/util/topsort.py index 7441af07516..80c6bd33962 100644 --- a/lib/galaxy/util/topsort.py +++ b/lib/galaxy/util/topsort.py @@ -197,7 +197,7 @@ def topsort_levels(pairlist): successors[first] = [second] answer = [] - + while 1: # Suck up everything without a predecessor. levparents = [x for x in numpreds.keys() if numpreds[x] == 0] @@ -210,10 +210,10 @@ def topsort_levels(pairlist): for levparentsucc in successors[levparent]: numpreds[levparentsucc] -= 1 del successors[levparent] - - if numpreds: - # Everything in num_parents has at least one child -> + + if numpreds: + # Everything in num_parents has at least one child -> # there's a cycle. raise CycleError( answer, numpreds, successors ) - + return answer diff --git a/lib/galaxy/visualization/data_providers/cigar.py b/lib/galaxy/visualization/data_providers/cigar.py index 993fc79b411..5727c9875ed 100644 --- a/lib/galaxy/visualization/data_providers/cigar.py +++ b/lib/galaxy/visualization/data_providers/cigar.py @@ -1,5 +1,5 @@ ''' -Functions for working with SAM/BAM CIGAR representation. +Functions for working with SAM/BAM CIGAR representation. ''' import operator @@ -8,9 +8,9 @@ def get_ref_based_read_seq_and_cigar( read_seq, read_start, ref_seq, ref_seq_sta ''' Returns a ( new_read_seq, new_cigar ) that can be used with reference sequence to reconstruct the read. The new read sequence includes only - bases that cannot be recovered from the reference: mismatches and + bases that cannot be recovered from the reference: mismatches and insertions (soft clipped bases are not included). The new cigar replaces - Ms with =s and Xs because the M operation can denote a sequence match or + Ms with =s and Xs because the M operation can denote a sequence match or mismatch. ''' @@ -48,7 +48,7 @@ def get_ref_based_read_seq_and_cigar( read_seq, read_start, ref_seq, ref_seq_sta while total_count < op_len and ref_seq_pos < len( ref_seq ): match, count = _match_mismatch_counter( read_seq, read_pos, ref_seq, ref_seq_pos ) # Use min because count cannot exceed remainder of operation. - count = min( count, op_len - total_count ) + count = min( count, op_len - total_count ) if match: new_op = '=' else: @@ -59,7 +59,7 @@ def get_ref_based_read_seq_and_cigar( read_seq, read_start, ref_seq, ref_seq_sta total_count += count read_pos += count ref_seq_pos += count - + # If end of read falls outside of ref_seq data, leave as M. if total_count < op_len: new_cigar += '%iM' % ( op_len - total_count ) diff --git a/lib/galaxy/visualization/data_providers/phyloviz/__init__.py b/lib/galaxy/visualization/data_providers/phyloviz/__init__.py index 85745da88ce..5d83e61ca5e 100644 --- a/lib/galaxy/visualization/data_providers/phyloviz/__init__.py +++ b/lib/galaxy/visualization/data_providers/phyloviz/__init__.py @@ -39,5 +39,5 @@ class PhylovizDataProvider( BaseDataProvider ): rval[ "data" ] = jsonDicts rval[ "msg"] = parseMsg - + return rval diff --git a/lib/galaxy/visualization/data_providers/phyloviz/phyloxmlparser.py b/lib/galaxy/visualization/data_providers/phyloviz/phyloxmlparser.py index 460de65435b..c5e38870bfb 100644 --- a/lib/galaxy/visualization/data_providers/phyloviz/phyloxmlparser.py +++ b/lib/galaxy/visualization/data_providers/phyloviz/phyloxmlparser.py @@ -130,4 +130,3 @@ class Phyloxml_Parser(Base_Parser): def cleanTag(self, tagString): return tagString[self.nameSpaceIndex:] - \ No newline at end of file diff --git a/lib/galaxy/visualization/data_providers/registry.py b/lib/galaxy/visualization/data_providers/registry.py index 50b96f0b800..6d159b73e04 100644 --- a/lib/galaxy/visualization/data_providers/registry.py +++ b/lib/galaxy/visualization/data_providers/registry.py @@ -17,13 +17,13 @@ class DataProviderRegistry( object ): # type. First key is converted dataset type; if result is another dict, second key # is original dataset type. self.dataset_type_name_to_data_provider = { - "tabix": { + "tabix": { Vcf: genome.VcfTabixDataProvider, Bed: genome.BedTabixDataProvider, Gtf: genome.GtfTabixDataProvider, ENCODEPeak: genome.ENCODEPeakTabixDataProvider, Interval: genome.IntervalTabixDataProvider, - ChromatinInteractions: genome.ChromatinInteractionsTabixDataProvider, + ChromatinInteractions: genome.ChromatinInteractionsTabixDataProvider, "default" : genome.TabixDataProvider }, "interval_index": genome.IntervalIndexDataProvider, @@ -37,7 +37,7 @@ class DataProviderRegistry( object ): def get_data_provider( self, trans, name=None, source='data', raw=False, original_dataset=None ): """ - Returns data provider matching parameter values. For standalone data + Returns data provider matching parameter values. For standalone data sources, source parameter is ignored. """ @@ -77,22 +77,22 @@ class DataProviderRegistry( object ): else: converted_dataset = original_dataset.get_converted_dataset( trans, name ) deps = original_dataset.get_converted_dataset_deps( trans, name ) - data_provider = data_provider_class( original_dataset=original_dataset, + data_provider = data_provider_class( original_dataset=original_dataset, converted_dataset=converted_dataset, dependencies=deps ) - + elif original_dataset: # No name, so look up a provider name from datatype's information. # Dataset must have data sources to get data. if not original_dataset.datatype.data_sources: return None - + # Get data provider mapping and data provider. data_provider_mapping = original_dataset.datatype.data_sources if 'data_standalone' in data_provider_mapping: - data_provider = self.get_data_provider( trans, - name=data_provider_mapping[ 'data_standalone' ], + data_provider = self.get_data_provider( trans, + name=data_provider_mapping[ 'data_standalone' ], original_dataset=original_dataset ) else: source_list = data_provider_mapping[ source ] diff --git a/lib/galaxy/visualization/genomes.py b/lib/galaxy/visualization/genomes.py index 4ce035ba273..74aa78ed8c0 100644 --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -30,12 +30,12 @@ class GenomeRegion( object ): """ A genomic region on an individual chromosome. """ - + def __init__( self, chrom = None, start = 0, end = 0 ): self.chrom = chrom self.start = int( start ) self.end = int( end ) - + def __str__( self ): return self.chrom + ":" + str( self.start ) + "-" + str( self.end ) @@ -44,22 +44,22 @@ class GenomeRegion( object ): return GenomeRegion( chrom = obj_dict[ 'chrom' ], start = obj_dict[ 'start' ], end = obj_dict[ 'end' ] ) - + @staticmethod def from_str( obj_str ): # check for gene region gene_region = obj_str.split(':') - + # split gene region into components if (len(gene_region) == 2): gene_interval = gene_region[1].split('-') - + # check length if (len(gene_interval) == 2): return GenomeRegion(chrom = gene_region[0], start = gene_interval[0], end = gene_interval[1]) - + # return genome region instance return GenomeRegion() @@ -72,12 +72,12 @@ class Genome( object ): self.description = description self.len_file = len_file self.twobit_file = twobit_file - + def dictify( self, num=None, chrom=None, low=None ): """ Returns representation of self as a dictionary. """ - + def check_int(s): if s.isdigit(): return int(s) @@ -86,7 +86,7 @@ class Genome( object ): def split_by_number(s): return [ check_int(c) for c in re.split('([0-9]+)', s) ] - + # # Parameter check, setting. # @@ -94,14 +94,14 @@ class Genome( object ): num = int( num ) else: num = sys.maxint - + if low: low = int( low ) if low < 0: low = 0 else: low = 0 - + # # Get chroms data: # (a) chrom name, len; @@ -117,7 +117,7 @@ class Genome( object ): found = False count = 0 for line_num, line in len_file_enumerate: - if line.startswith("#"): + if line.startswith("#"): continue name, len = line.split("\t") if found: @@ -133,25 +133,25 @@ class Genome( object ): prev_chroms = True if count >= num: break - else: + else: # Use low to start list. high = low + int( num ) prev_chroms = ( low != 0 ) start_index = low - + # Read chrom data from len file. for line_num, line in len_file_enumerate: if line_num < low: continue if line_num >= high: break - if line.startswith("#"): + if line.startswith("#"): continue # LEN files have format: # fields = line.split("\t") chroms[ fields[0] ] = int( fields[1] ) - + # Set flag to indicate whether there are more chroms after list. next_chroms = False try: @@ -160,23 +160,23 @@ class Genome( object ): except: # No more chroms to read. pass - + to_sort = [{ 'chrom': chrom, 'len': length } for chrom, length in chroms.iteritems()] to_sort.sort(lambda a,b: cmp( split_by_number(a['chrom']), split_by_number(b['chrom']) )) return { 'id': self.key, - 'reference': self.twobit_file is not None, - 'chrom_info': to_sort, - 'prev_chroms' : prev_chroms, - 'next_chroms' : next_chroms, + 'reference': self.twobit_file is not None, + 'chrom_info': to_sort, + 'prev_chroms' : prev_chroms, + 'next_chroms' : next_chroms, 'start_index' : start_index } - + class Genomes( object ): """ Provides information about available genome data and methods for manipulating that data. """ - + def __init__( self, app ): # Create list of genomes from util.dbnames self.genomes = {} @@ -189,7 +189,7 @@ class Genomes( object ): key = os.path.split( f )[1].split( ".len" )[0] if key in self.genomes: self.genomes[ key ].len_file = f - + # Add genome data (twobit files) to genomes. try: for line in open( os.path.join( app.config.tool_data_path, "twobit.loc" ) ): @@ -202,16 +202,16 @@ class Genomes( object ): except IOError, e: # Thrown if twobit.loc does not exist. log.exception( str( e ) ) - + def get_build( self, dbkey ): """ Returns build for the given key. """ rval = None if dbkey in self.genomes: rval = self.genomes[ dbkey ] return rval - + def get_dbkeys( self, trans, chrom_info=False ): - """ Returns all known dbkeys. If chrom_info is True, only dbkeys with + """ Returns all known dbkeys. If chrom_info is True, only dbkeys with chromosome lengths are returned. """ dbkeys = [] @@ -230,23 +230,23 @@ class Genomes( object ): filter_fn = lambda b: b.len_file is not None dbkeys.extend( [ ( genome.description, genome.key ) for key, genome in self.genomes.items() if filter_fn( genome ) ] ) - + return dbkeys - - + + def chroms( self, trans, dbkey=None, num=None, chrom=None, low=None ): """ Returns a naturally sorted list of chroms/contigs for a given dbkey. Use either chrom or low to specify the starting chrom in the return list. """ - + # If there is no dbkey owner, default to current user. dbkey_owner, dbkey = decode_dbkey( dbkey ) if dbkey_owner: dbkey_user = trans.sa_session.query( trans.app.model.User ).filter_by( username=dbkey_owner ).first() else: dbkey_user = trans.user - + # # Get/create genome object. # @@ -265,7 +265,7 @@ class Genomes( object ): build_fasta = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dbkey_attributes[ 'fasta' ] ) len_file = build_fasta.get_converted_dataset( trans, 'len' ).file_name build_fasta.get_converted_dataset( trans, 'twobit' ) - # HACK: set twobit_file to True rather than a file name because + # HACK: set twobit_file to True rather than a file name because # get_converted_dataset returns null during conversion even though # there will eventually be a twobit file available for genome. twobit_file = True @@ -274,8 +274,8 @@ class Genomes( object ): len_file = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( user_keys[ dbkey ][ 'len' ] ).file_name if len_file: genome = Genome( dbkey, dbkey_name, len_file=len_file, twobit_file=twobit_file ) - - + + # Look in history and system builds. if not genome: # Look in history for chromosome len file. @@ -292,13 +292,13 @@ class Genomes( object ): rval = genome.dictify( num=num, chrom=chrom, low=low ) else: log.exception( 'genome not found for key %s' % dbkey ) - + return rval - + def has_reference_data( self, dbkey, dbkey_owner=None ): - """ - Returns true if there is reference data for the specified dbkey. If dbkey is custom, + """ + Returns true if there is reference data for the specified dbkey. If dbkey is custom, dbkey_owner is needed to determine if there is reference data. """ # Look for key in built-in builds. @@ -316,7 +316,7 @@ class Genomes( object ): return True return False - + def reference( self, trans, dbkey, chrom, low, high ): """ Return reference data for a build. @@ -332,7 +332,7 @@ class Genomes( object ): if not self.has_reference_data( dbkey, dbkey_user ): return None - # + # # Get twobit file with reference data. # twobit_file_name = None diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index 1e27886d6c6..c691b593544 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -663,7 +663,7 @@ class UsesHistoryDatasetAssociationMixin: display_apps = [] if not trans.app.config.enable_old_display_applications: return display_apps - + for display_app in hda.datatype.get_display_types(): target_frame, display_links = hda.datatype.get_display_links( hda, display_app, trans.app, trans.request.base ) @@ -1123,7 +1123,7 @@ class UsesVisualizationMixin( UsesHistoryDatasetAssociationMixin, UsesLibraryMix tool = trans.app.toolbox.get_tool( job.tool_id ) if not tool: return None - + # Tool must have a Trackster configuration. if not tool.trackster_conf: return None @@ -1142,7 +1142,7 @@ class UsesVisualizationMixin( UsesHistoryDatasetAssociationMixin, UsesLibraryMix if isinstance( value, DictifiableMixin ): value = value.dictify() t_input[ 'value' ] = value - + return tool_dict def get_visualization_config( self, trans, visualization ): @@ -2328,16 +2328,16 @@ class UsesTagsMixin( object ): class UsesExtendedMetadataMixin( SharableItemSecurityMixin ): """ Mixin for getting and setting item extended metadata. """ - + def get_item_extended_metadata_obj( self, trans, item ): """ - Given an item object (such as a LibraryDatasetDatasetAssociation), find the object + Given an item object (such as a LibraryDatasetDatasetAssociation), find the object of the associated extended metadata """ if item.extended_metadata: return item.extended_metadata return None - + def set_item_extended_metadata_obj( self, trans, item, extmeta_obj, check_writable=False): print "setting", extmeta_obj.data if item.__class__ == LibraryDatasetDatasetAssociation: diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index e199aca68ee..074c2827470 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -23,7 +23,7 @@ class BaseField(object): class TextField(BaseField): """ A standard text input box. - + >>> print TextField( "foo" ).get_html() >>> print TextField( "bins", size=4, value="default" ).get_html() @@ -42,11 +42,11 @@ class TextField(BaseField): % ( prefix, self.name, self.size, escape( value, quote=True ), self.get_disabled_str( disabled ) ) ) def set_size(self, size): self.size = int( size ) - + class PasswordField(BaseField): """ A password input box. text appears as "******" - + >>> print PasswordField( "foo" ).get_html() >>> print PasswordField( "bins", size=4, value="default" ).get_html() @@ -65,7 +65,7 @@ class PasswordField(BaseField): class TextArea(BaseField): """ A standard text area box. - + >>> print TextArea( "foo" ).get_html() >>> print TextArea( "bins", size="4x5", value="default" ).get_html() @@ -89,7 +89,7 @@ class TextArea(BaseField): class CheckboxField(BaseField): """ A checkbox (boolean input) - + >>> print CheckboxField( "foo" ).get_html() >>> print CheckboxField( "bar", checked="yes" ).get_html() @@ -100,7 +100,7 @@ class CheckboxField(BaseField): self.checked = ( checked == True ) or ( isinstance( checked, basestring ) and ( checked.lower() in ( "yes", "true", "on" ) ) ) self.refresh_on_change = refresh_on_change self.refresh_on_change_values = refresh_on_change_values or [] - if self.refresh_on_change: + if self.refresh_on_change: self.refresh_on_change_text = ' refresh_on_change="true" ' if self.refresh_on_change_values: self.refresh_on_change_text = '%s refresh_on_change_values="%s" ' % ( self.refresh_on_change_text, ",".join( self.refresh_on_change_values ) ) @@ -137,7 +137,7 @@ class CheckboxField(BaseField): class FileField(BaseField): """ A file upload input. - + >>> print FileField( "foo" ).get_html() >>> print FileField( "foo", ajax = True ).get_html() @@ -161,7 +161,7 @@ class FTPFileField(BaseField): An FTP file upload input. """ thead = ''' - +
    @@ -221,7 +221,7 @@ class FTPFileField(BaseField): class HiddenField(BaseField): """ A hidden field. - + >>> print HiddenField( "foo", 100 ).get_html() """ @@ -234,7 +234,7 @@ class HiddenField(BaseField): class SelectField(BaseField): """ A select field. - + >>> t = SelectField( "foo", multiple=True ) >>> t.add_option( "tuti", 1 ) >>> t.add_option( "fruity", "x" ) @@ -243,7 +243,7 @@ class SelectField(BaseField): - + >>> t = SelectField( "bar" ) >>> t.add_option( "automatic", 3 ) >>> t.add_option( "bazooty", 4, selected=True ) @@ -252,7 +252,7 @@ class SelectField(BaseField): - + >>> t = SelectField( "foo", display="radio" ) >>> t.add_option( "tuti", 1 ) >>> t.add_option( "fruity", "x" ) @@ -282,7 +282,7 @@ class SelectField(BaseField): self.display = display self.refresh_on_change = refresh_on_change self.refresh_on_change_values = refresh_on_change_values or [] - if self.refresh_on_change: + if self.refresh_on_change: self.refresh_on_change_text = ' refresh_on_change="true"' if self.refresh_on_change_values: self.refresh_on_change_text = '%s refresh_on_change_values="%s"' % ( self.refresh_on_change_text, escape( ",".join( self.refresh_on_change_values ), quote=True ) ) @@ -344,7 +344,7 @@ class SelectField(BaseField): uniq_id, text ) ) ctr += 1 - return unicodify( "\n".join( rval ) ) + return unicodify( "\n".join( rval ) ) def get_html_default( self, prefix="", disabled=False ): if self.multiple: multiple = " multiple" @@ -406,7 +406,7 @@ class SelectField(BaseField): class DrillDownField( BaseField ): """ A hierarchical select field, which allows users to 'drill down' a tree-like set of options. - + >>> t = DrillDownField( "foo", multiple=True, display="checkbox", options=[{'name': 'Heading 1', 'value': 'heading1', 'options': [{'name': 'Option 1', 'value': 'option1', 'options': []}, {'name': 'Option 2', 'value': 'option2', 'options': []}, {'name': 'Heading 1', 'value': 'heading1', 'options': [{'name': 'Option 3', 'value': 'option3', 'options': []}, {'name': 'Option 4', 'value': 'option4', 'options': []}]}]}, {'name': 'Option 5', 'value': 'option5', 'options': []}] ) >>> print t.get_html()
    @@ -488,7 +488,7 @@ class DrillDownField( BaseField ): self.display = display self.refresh_on_change = refresh_on_change self.refresh_on_change_values = refresh_on_change_values - if self.refresh_on_change: + if self.refresh_on_change: self.refresh_on_change_text = ' refresh_on_change="true"' if self.refresh_on_change_values: self.refresh_on_change_text = '%s refresh_on_change_values="%s"' % ( self.refresh_on_change_text, ",".join( self.refresh_on_change_values ) ) @@ -531,7 +531,7 @@ class DrillDownField( BaseField ): recurse_options( rval, self.options, drilldown_id, expanded_options ) rval.append( '
    ' ) return unicodify( '\n'.join( rval ) ) - + class AddressField(BaseField): @staticmethod def fields(): @@ -557,8 +557,8 @@ class AddressField(BaseField): for a in self.user.addresses: add_ids.append( str( a.id ) ) add_ids.append( 'new' ) - self.select_address = SelectField( self.name, - refresh_on_change=True, + self.select_address = SelectField( self.name, + refresh_on_change=True, refresh_on_change_values=add_ids ) if self.value == 'none': self.select_address.add_option( 'Select one', 'none', selected=True ) @@ -580,9 +580,9 @@ class AddressField(BaseField): if self.value == 'new': self.select_address.add_option( 'Add a new address', 'new', selected=True ) for field_name, label, help_text in self.fields(): - add_field = TextField( self.name + '_' + field_name, + add_field = TextField( self.name + '_' + field_name, 40, - restore_text( self.params.get( self.name + '_' + field_name, '' ) ) ) + restore_text( self.params.get( self.name + '_' + field_name, '' ) ) ) address_html += '''
    @@ -593,7 +593,7 @@ class AddressField(BaseField):
    %s
    - ''' % help_text + ''' % help_text address_html += '''
    ''' @@ -692,7 +692,7 @@ class HistoryField( BaseField ): return self.value else: return '-' - + class LibraryField( BaseField ): def __init__( self, name, value=None, trans=None ): self.name = name @@ -727,7 +727,7 @@ def build_select_field( trans, objs, label_attr, select_field_name, initial_val Build a SelectField given a set of objects. The received params are: - objs: the set of objects used to populate the option list - - label_attr: the attribute of each obj (e.g., name, email, etc ) whose value is used to populate each option label. + - label_attr: the attribute of each obj (e.g., name, email, etc ) whose value is used to populate each option label. - If the string 'self' is passed as label_attr, each obj in objs is assumed to be a string, so the obj itself is used @@ -750,10 +750,10 @@ def build_select_field( trans, objs, label_attr, select_field_name, initial_val refresh_on_change_values = values else: refresh_on_change_values = [] - select_field = SelectField( name=select_field_name, + select_field = SelectField( name=select_field_name, multiple=multiple, display=display, - refresh_on_change=refresh_on_change, + refresh_on_change=refresh_on_change, refresh_on_change_values=refresh_on_change_values, size=size ) if display is None and initial_value == 'none': diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index 82b34131080..2e87a841130 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -270,7 +270,7 @@ class WebApplication( base.WebApplication ): def add_ui_controllers( self, package_name, app ): """ - Search for UI controllers in `package_name` and add + Search for UI controllers in `package_name` and add them to the webapp. """ from galaxy.web.base.controller import BaseUIController @@ -294,7 +294,7 @@ class WebApplication( base.WebApplication ): def add_api_controllers( self, package_name, app ): """ - Search for UI controllers in `package_name` and add + Search for UI controllers in `package_name` and add them to the webapp. """ from galaxy.web.base.controller import BaseAPIController @@ -842,10 +842,10 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ): if not self.galaxy_session.user: return self.new_history() - # Look for default history that (a) has default name + is not deleted and - # (b) has no datasets. If suitable history found, use it; otherwise, create + # Look for default history that (a) has default name + is not deleted and + # (b) has no datasets. If suitable history found, use it; otherwise, create # new history. - unnamed_histories = self.sa_session.query( self.app.model.History ).filter_by( + unnamed_histories = self.sa_session.query( self.app.model.History ).filter_by( user=self.galaxy_session.user, name=self.app.model.History.default_name, deleted=False ) @@ -855,7 +855,7 @@ class GalaxyWebTransaction( base.DefaultWebTransaction ): # Found suitable default history. default_history = history break - + # Set or create hsitory. if default_history: history = default_history diff --git a/lib/galaxy/web/framework/helpers/__init__.py b/lib/galaxy/web/framework/helpers/__init__.py index e28259ec60a..3d28eb951d2 100644 --- a/lib/galaxy/web/framework/helpers/__init__.py +++ b/lib/galaxy/web/framework/helpers/__init__.py @@ -17,19 +17,19 @@ def time_ago( x ): Convert a datetime to a string. """ delta = timedelta(weeks=1) - + # If the date is more than one week ago, then display the actual date instead of in words if (datetime.utcnow() - x) > delta: # Greater than a week difference return x.strftime("%b %d, %Y") - else: + else: return date.distance_of_time_in_words( x, datetime.utcnow() ).replace("about", "~") + " ago" - + def iff( a, b, c ): if a: return b else: return c - + def truncate(content, length=100, suffix='...'): """ Smart string truncation @@ -38,18 +38,18 @@ def truncate(content, length=100, suffix='...'): return content else: return content[:length].rsplit(' ', 1)[0] + suffix - + # Quick helpers for static content def css( *args ): """ Take a list of stylesheet names (no extension) and return appropriate string of link tags. - + Cache-bust with time that server started running on """ return "\n".join( [ stylesheet_link_tag( "/static/style/" + name + ".css?v=%s" % server_starttime ) for name in args ] ) - + def js_helper( prefix, *args ): """ Take a prefix and list of javascript names and return appropriate @@ -58,21 +58,21 @@ def js_helper( prefix, *args ): Cache-bust with time that server started running on """ return "\n".join( [ javascript_include_tag( prefix + name + ".js?v=%s" % server_starttime ) for name in args ] ) - + def js( *args ): """ Take a prefix and list of javascript names and return appropriate string of script tags. """ return js_helper( '/static/scripts/', *args ) - + def templates( *args ): """ Take a list of template names (no extension) and return appropriate string of script tags. """ return js_helper( '/static/scripts/templates/compiled/', *args ) - + # Hashes def md5( s ): @@ -82,11 +82,11 @@ def md5( s ): m = hash_util.md5() m.update( s ) return m.hexdigest() - + # Unicode help def to_unicode( a_string ): - """ + """ Convert a string to unicode in utf-8 format; if string is already unicode, does nothing because string's encoding cannot be determined by introspection. """ @@ -102,4 +102,3 @@ def is_true ( val ): """ return val == True or val in [ 'True', 'true', 'T', 't' ] - \ No newline at end of file diff --git a/lib/galaxy/web/framework/helpers/grids.py b/lib/galaxy/web/framework/helpers/grids.py index 30a87251c6a..8f480952381 100644 --- a/lib/galaxy/web/framework/helpers/grids.py +++ b/lib/galaxy/web/framework/helpers/grids.py @@ -281,7 +281,7 @@ class Grid( object ): status = status, message = message, use_panels=self.use_panels, - show_item_checkboxes = ( self.show_item_checkboxes or + show_item_checkboxes = ( self.show_item_checkboxes or kwargs.get( 'show_item_checkboxes', '' ) in [ 'True', 'true' ] ), # Pass back kwargs so that grid template can set and use args without # grid explicitly having to pass them. diff --git a/lib/galaxy/web/framework/middleware/error.py b/lib/galaxy/web/framework/middleware/error.py index 5b45837cdc5..fa7f7c7c03e 100644 --- a/lib/galaxy/web/framework/middleware/error.py +++ b/lib/galaxy/web/framework/middleware/error.py @@ -31,7 +31,7 @@ class ErrorMiddleware(object): """ Error handling middleware - + Usage:: error_catching_wsgi_app = ErrorMiddleware(wsgi_app) @@ -42,14 +42,14 @@ class ErrorMiddleware(object): If true, then tracebacks will be shown in the browser. ``error_email``: - an email address (or list of addresses) to send exception + an email address (or list of addresses) to send exception reports to ``error_log``: a filename to append tracebacks to ``show_exceptions_in_wsgi_errors``: - If true, then errors will be printed to ``wsgi.errors`` + If true, then errors will be printed to ``wsgi.errors`` (frequently a server error log, or stderr). ``from_address``, ``smtp_server``, ``error_subject_prefix``, ``smtp_username``, ``smtp_password``, ``smtp_use_tls``: @@ -65,7 +65,7 @@ class ErrorMiddleware(object): HTML page. Environment Configuration: - + ``paste.throw_errors``: If this setting in the request environment is true, then this middleware is disabled. This can be useful in a testing situation @@ -73,10 +73,10 @@ class ErrorMiddleware(object): ``paste.expected_exceptions``: When this middleware encounters an exception listed in this - environment variable and when the ``start_response`` has not + environment variable and when the ``start_response`` has not yet occurred, the exception will be re-raised instead of being - caught. This should generally be set by middleware that may - (but probably shouldn't be) installed above this middleware, + caught. This should generally be set by middleware that may + (but probably shouldn't be) installed above this middleware, and wants to get certain exceptions. Exceptions raised after ``start_response`` have been called are always caught since by definition they are no longer expected. @@ -131,7 +131,7 @@ class ErrorMiddleware(object): if xmlhttp_key is None: xmlhttp_key = global_conf.get('xmlhttp_key', '_') self.xmlhttp_key = xmlhttp_key - + def __call__(self, environ, start_response): """ The WSGI application interface. @@ -200,7 +200,7 @@ class ResponseStartChecker(object): def __call__(self, *args): self.response_started = True - # Return whatever the wrapped start_response would have + # Return whatever the wrapped start_response would have # returned return self.start_response(*args) @@ -323,7 +323,7 @@ class Supplement(object): (1, 0, 1): 'CGI', (1, 1, 1): 'Multi thread/process CGI (?)', } - + def handle_exception(exc_info, error_stream, html=True, debug_mode=False, error_email=None, @@ -331,8 +331,8 @@ def handle_exception(exc_info, error_stream, html=True, show_exceptions_in_wsgi_errors=False, error_email_from='errors@localhost', smtp_server='localhost', - smtp_username=None, - smtp_password=None, + smtp_username=None, + smtp_password=None, smtp_use_tls=False, error_subject_prefix='', error_message=None, diff --git a/lib/galaxy/web/framework/middleware/profile.py b/lib/galaxy/web/framework/middleware/profile.py index 32fdb47d575..dc276a8bcd5 100644 --- a/lib/galaxy/web/framework/middleware/profile.py +++ b/lib/galaxy/web/framework/middleware/profile.py @@ -12,7 +12,7 @@ from cStringIO import StringIO from paste import response try: - # Included in Python 2.5 + # Included in Python 2.5 import cProfile except: try: @@ -91,7 +91,7 @@ class ProfileMiddleware(object): output = pstats_as_html( stats, self.limit ) body += template % output return [body] - + def pstats_as_html( stats, *sel_list ): """ Return an HTML representation of a pstats.Stats object. @@ -149,15 +149,15 @@ def pstats_as_html( stats, *sel_list ): rval.append( "
    ") # Concatenate result return "".join( rval ) - + def get_func_list( stats, sel_list ): """ - Use 'sel_list' to select a list of functions to display. + Use 'sel_list' to select a list of functions to display. """ # Determine if an ordering was applied if stats.fcn_list: list = stats.fcn_list[:] - order_message = "Ordered by: " + stats.sort_type + order_message = "Ordered by: " + stats.sort_type else: list = stats.stats.keys() order_message = "Random listing order was used" @@ -167,8 +167,8 @@ def get_func_list( stats, sel_list ): list, select_message = stats.eval_print_amount( selection, list, select_message ) # Return the list of functions selected and the message return list, order_message, select_message - -def func_std_string( func_name ): + +def func_std_string( func_name ): """ Match what old profile produced """ diff --git a/lib/galaxy/web/framework/middleware/sentry.py b/lib/galaxy/web/framework/middleware/sentry.py index 437ba63fc39..6198f13161d 100644 --- a/lib/galaxy/web/framework/middleware/sentry.py +++ b/lib/galaxy/web/framework/middleware/sentry.py @@ -6,7 +6,7 @@ raven.middleware :license: BSD, see LICENSE for more details. """ -import galaxy.eggs; galaxy.eggs.require( "raven" ) +import galaxy.eggs; galaxy.eggs.require( "raven" ) from raven import Client from raven.utils.wsgi import get_current_url, get_headers, \ diff --git a/lib/galaxy/web/framework/middleware/xforwardedhost.py b/lib/galaxy/web/framework/middleware/xforwardedhost.py index ce149ed3b6a..89dc18089e5 100644 --- a/lib/galaxy/web/framework/middleware/xforwardedhost.py +++ b/lib/galaxy/web/framework/middleware/xforwardedhost.py @@ -1,7 +1,7 @@ class XForwardedHostMiddleware( object ): """ A WSGI middleware that changes the HTTP host header in the WSGI environ - based on the X-Forwarded-Host header IF found + based on the X-Forwarded-Host header IF found """ def __init__( self, app, global_conf=None ): self.app = app diff --git a/lib/galaxy/web/security/__init__.py b/lib/galaxy/web/security/__init__.py index b62f989dcdd..4120de8c9e7 100644 --- a/lib/galaxy/web/security/__init__.py +++ b/lib/galaxy/web/security/__init__.py @@ -40,7 +40,7 @@ class SecurityHelper( object ): def encode_id( self, obj_id ): # Convert to string s = str( obj_id ) - # Pad to a multiple of 8 with leading "!" + # Pad to a multiple of 8 with leading "!" s = ( "!" * ( 8 - len(s) % 8 ) ) + s # Encrypt return self.id_cipher.encrypt( s ).encode( 'hex' ) @@ -61,7 +61,7 @@ class SecurityHelper( object ): def encode_guid( self, session_key ): # Session keys are strings - # Pad to a multiple of 8 with leading "!" + # Pad to a multiple of 8 with leading "!" s = ( "!" * ( 8 - len( session_key ) % 8 ) ) + session_key # Encrypt return self.id_cipher.encrypt( s ).encode( 'hex' ) diff --git a/lib/galaxy/webapps/demo_sequencer/buildapp.py b/lib/galaxy/webapps/demo_sequencer/buildapp.py index 57e95cee263..e38dea7a484 100644 --- a/lib/galaxy/webapps/demo_sequencer/buildapp.py +++ b/lib/galaxy/webapps/demo_sequencer/buildapp.py @@ -68,7 +68,7 @@ def app_factory( global_conf, **kwargs ): webapp = wrap_in_static( webapp, global_conf, **kwargs ) # Return return webapp - + def wrap_in_middleware( app, global_conf, **local_conf ): """Based on the configuration wrap `app` in a set of common and useful middleware.""" # Merge the global and local configurations @@ -80,7 +80,7 @@ def wrap_in_middleware( app, global_conf, **local_conf ): # other middleware): app = httpexceptions.make_middleware( app, conf ) log.debug( "Enabling 'httpexceptions' middleware" ) - # The recursive middleware allows for including requests in other + # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): from paste import recursive @@ -130,7 +130,7 @@ def wrap_in_middleware( app, global_conf, **local_conf ): app = XForwardedHostMiddleware( app ) log.debug( "Enabling 'x-forwarded-host' middleware" ) return app - + def wrap_in_static( app, global_conf, **local_conf ): from paste.urlmap import URLMap from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static @@ -152,7 +152,7 @@ def wrap_in_static( app, global_conf, **local_conf ): urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir" ), cache_time ) # URL mapper becomes the root webapp return urlmap - + def build_template_error_formatters(): """ Build a list of template error formatters for WebError. When an error diff --git a/lib/galaxy/webapps/demo_sequencer/config.py b/lib/galaxy/webapps/demo_sequencer/config.py index 5fb8420b04d..72429497913 100644 --- a/lib/galaxy/webapps/demo_sequencer/config.py +++ b/lib/galaxy/webapps/demo_sequencer/config.py @@ -107,7 +107,7 @@ def configure_logging( config ): if level <= logging.DEBUG: logging.getLogger( "paste.httpserver.ThreadPool" ).setLevel( logging.WARN ) # Remove old handlers - for h in root.handlers[:]: + for h in root.handlers[:]: root.removeHandler(h) # Create handler if destination == "stdout": @@ -115,7 +115,7 @@ def configure_logging( config ): else: handler = logging.FileHandler( destination ) # Create formatter - formatter = logging.Formatter( format ) + formatter = logging.Formatter( format ) # Hook everything up handler.setFormatter( formatter ) root.addHandler( handler ) diff --git a/lib/galaxy/webapps/demo_sequencer/framework/__init__.py b/lib/galaxy/webapps/demo_sequencer/framework/__init__.py index 41328b0b4a9..54cc7e121cf 100644 --- a/lib/galaxy/webapps/demo_sequencer/framework/__init__.py +++ b/lib/galaxy/webapps/demo_sequencer/framework/__init__.py @@ -32,7 +32,7 @@ pkg_resources.require( "amqplib" ) import logging log = logging.getLogger( __name__ ) - + class WebApplication( galaxy.web.framework.base.WebApplication ): def __init__( self, demo_app, session_cookie='demosequencersession' ): galaxy.web.framework.base.WebApplication.__init__( self ) @@ -81,13 +81,13 @@ class DemoWebTransaction( galaxy.web.framework.base.DefaultWebTransaction ): return None def set_cookie( self, value, name='demosequencersession', path='/', age=90, version='1' ): """Convenience method for setting a session cookie""" - # The demosequencersession cookie value must be a high entropy 128 bit random number encrypted + # The demosequencersession cookie value must be a high entropy 128 bit random number encrypted # using a server secret key. Any other value is invalid and could pose security issues. self.response.cookies[name] = value self.response.cookies[name]['path'] = path self.response.cookies[name]['max-age'] = 3600 * 24 * age # 90 days tstamp = time.localtime ( time.time() + 3600 * 24 * age ) - self.response.cookies[name]['expires'] = time.strftime( '%a, %d-%b-%Y %H:%M:%S GMT', tstamp ) + self.response.cookies[name]['expires'] = time.strftime( '%a, %d-%b-%Y %H:%M:%S GMT', tstamp ) self.response.cookies[name]['version'] = version def __update_session_cookie( self, name='galaxysession' ): """ @@ -108,7 +108,7 @@ class DemoWebTransaction( galaxy.web.framework.base.DefaultWebTransaction ): return rval def set_message( self, message, type=None ): """ - Convenience method for setting the 'message' and 'message_type' + Convenience method for setting the 'message' and 'message_type' element of the template context. """ self.template_context['message'] = message @@ -123,11 +123,11 @@ class DemoWebTransaction( galaxy.web.framework.base.DefaultWebTransaction ): def show_message( self, message, type='info', refresh_frames=[], cont=None, use_panels=False, active_view="" ): """ Convenience method for displaying a simple page with a single message. - + `type`: one of "error", "warning", "info", or "done"; determines the type of dialog box and icon displayed with the message - - `refresh_frames`: names of frames in the interface that should be + + `refresh_frames`: names of frames in the interface that should be refreshed when the message is displayed """ return self.fill_template( "message.mako", status=type, message=message, refresh_frames=refresh_frames, cont=cont, use_panels=use_panels, active_view=active_view ) @@ -151,7 +151,7 @@ class DemoWebTransaction( galaxy.web.framework.base.DefaultWebTransaction ): Convenience method for displaying a simple page with a single HTML form. """ - return self.fill_template( template, form=form, header=header, use_panels=( form.use_panels or use_panels ), + return self.fill_template( template, form=form, header=header, use_panels=( form.use_panels or use_panels ), active_view=active_view ) def fill_template(self, filename, **kwargs): """ @@ -160,19 +160,19 @@ class DemoWebTransaction( galaxy.web.framework.base.DefaultWebTransaction ): if filename.endswith( ".mako" ): return self.fill_template_mako( filename, **kwargs ) else: - template = Template( file=os.path.join(self.app.config.template_path, filename), + template = Template( file=os.path.join(self.app.config.template_path, filename), searchList=[kwargs, self.template_context, dict(caller=self, t=self, h=helpers, util=util, request=self.request, response=self.response, app=self.app)] ) return str( template ) def fill_template_mako( self, filename, **kwargs ): template = self.webapp.mako_template_lookup.get_template( filename ) - template.output_encoding = 'utf-8' + template.output_encoding = 'utf-8' data = dict( caller=self, t=self, trans=self, h=helpers, util=util, request=self.request, response=self.response, app=self.app ) data.update( self.template_context ) data.update( kwargs ) return template.render( **data ) def stream_template_mako( self, filename, **kwargs ): template = self.webapp.mako_template_lookup.get_template( filename ) - template.output_encoding = 'utf-8' + template.output_encoding = 'utf-8' data = dict( caller=self, t=self, trans=self, h=helpers, util=util, request=self.request, response=self.response, app=self.app ) data.update( self.template_context ) data.update( kwargs ) @@ -193,7 +193,7 @@ class DemoWebTransaction( galaxy.web.framework.base.DefaultWebTransaction ): """ template = Template( source=template_string, searchList=[context or kwargs, dict(caller=self)] ) - return str(template) + return str(template) class DemoWebAPITransaction( DemoWebTransaction ): def __init__( self, environ, app, webapp ): diff --git a/lib/galaxy/webapps/demo_sequencer/registry.py b/lib/galaxy/webapps/demo_sequencer/registry.py index 6018db8d000..ca7c772aabb 100644 --- a/lib/galaxy/webapps/demo_sequencer/registry.py +++ b/lib/galaxy/webapps/demo_sequencer/registry.py @@ -72,7 +72,7 @@ class Registry( object ): for redirect_elem in root.findall( 'redirect' ): requests = [] action_dict = {} - action_dict[ 'title' ] = redirect_elem.get( 'title', None ) + action_dict[ 'title' ] = redirect_elem.get( 'title', None ) action = redirect_elem.get( 'action', None ) # Load the external webapp requests, if any exist for this redirect action for request_elem in redirect_elem.findall( 'external_webapp' ): diff --git a/lib/galaxy/webapps/galaxy/api/annotations.py b/lib/galaxy/webapps/galaxy/api/annotations.py index 3960ec6ee45..2b46bac36cf 100644 --- a/lib/galaxy/webapps/galaxy/api/annotations.py +++ b/lib/galaxy/webapps/galaxy/api/annotations.py @@ -4,7 +4,7 @@ API operations on annotations. import logging, os, string, shutil, urllib, re, socket from cgi import escape, FieldStorage from galaxy import util, datatypes, jobs, web, util -from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin, UsesHistoryDatasetAssociationMixin, UsesStoredWorkflowMixin +from galaxy.web.base.controller import BaseAPIController, UsesHistoryMixin, UsesHistoryDatasetAssociationMixin, UsesStoredWorkflowMixin from galaxy.model.item_attrs import UsesAnnotations from galaxy.util.sanitize_html import sanitize_html import galaxy.datatypes @@ -27,7 +27,7 @@ class BaseAnnotationsController( BaseAPIController, UsesAnnotations, UsesHistory @web.expose_api - def create( self, trans, payload, **kwd ): + def create( self, trans, payload, **kwd ): if "text" not in payload: return "" idnum = kwd[self.tagged_item_id] diff --git a/lib/galaxy/webapps/galaxy/api/datasets.py b/lib/galaxy/webapps/galaxy/api/datasets.py index e7457b55450..c911ff55aae 100644 --- a/lib/galaxy/webapps/galaxy/api/datasets.py +++ b/lib/galaxy/webapps/galaxy/api/datasets.py @@ -22,7 +22,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory """ trans.response.status = 501 return 'not implemented' - + @web.expose_api def show( self, trans, id, hda_ldda='hda', data_type=None, provider=None, **kwd ): """ @@ -40,7 +40,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory if data_type == 'state': rval = self._dataset_state( trans, dataset ) elif data_type == 'converted_datasets_state': - rval = self._converted_datasets_state( trans, dataset, kwd.get( 'chrom', None ), + rval = self._converted_datasets_state( trans, dataset, kwd.get( 'chrom', None ), is_true( kwd.get( 'retry', False ) ) ) elif data_type == 'data': rval = self._data( trans, dataset, **kwd ) @@ -61,7 +61,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory rval[ 'display_apps' ] = self.get_display_apps( trans, dataset ) else: rval = dataset.dictify() - + except Exception, e: rval = "Error in dataset API at listing contents: " + str( e ) log.error( rval + ": %s" % str(e), exc_info=True ) @@ -77,16 +77,16 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory msg = dataset.conversion_messages.DATA return msg - + def _converted_datasets_state( self, trans, dataset, chrom=None, retry=False ): """ - Init-like method that returns state of dataset's converted datasets. + Init-like method that returns state of dataset's converted datasets. Returns valid chroms for that dataset as well. """ msg = self.check_dataset_state( trans, dataset ) if msg: return msg - + # Get datasources and check for messages (which indicate errors). Retry if flag is set. data_sources = dataset.get_datasources( trans ) messages_list = [ data_source_dict[ 'message' ] for data_source_dict in data_sources.values() ] @@ -98,7 +98,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory return self._converted_datasets_state( trans, dataset, chrom ) else: return msg - + # If there is a chrom, check for data on the chrom. if chrom: data_provider_registry = trans.app.data_provider_registry @@ -112,7 +112,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory def _search_features( self, trans, dataset, query ): """ - Returns features, locations in dataset that match query. Format is a + Returns features, locations in dataset that match query. Format is a list of features; each feature is a list itself: [name, location] """ if dataset.can_convert_to( "fli" ): @@ -121,9 +121,9 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory data_provider = FeatureLocationIndexDataProvider( converted_dataset=converted_dataset ) if data_provider: return data_provider.get_data( query ) - + return [] - + def _data( self, trans, dataset, chrom, low, high, start_val=0, max_vals=None, **kwargs ): """ Provides a block of data from a dataset. @@ -131,19 +131,19 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory # Parameter check. if not chrom: return dataset.conversion_messages.NO_DATA - + # Dataset check. msg = self.check_dataset_state( trans, dataset ) if msg: return msg - + # Get datasources and check for essages. data_sources = dataset.get_datasources( trans ) messages_list = [ data_source_dict[ 'message' ] for data_source_dict in data_sources.values() ] return_message = self._get_highest_priority_msg( messages_list ) if return_message: return return_message - + extra_info = None mode = kwargs.get( "mode", "Auto" ) data_provider_registry = trans.app.data_provider_registry @@ -154,10 +154,10 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory indexer = data_provider_registry.get_data_provider( trans, original_dataset=dataset, source='index' ) return indexer.get_data( chrom, low, high, **kwargs ) - # TODO: + # TODO: # (1) add logic back in for no_detail # (2) handle scenario where mode is Squish/Pack but data requested is large, so reduced data needed to be returned. - + # If mode is Auto, need to determine what type of data to return. if mode == "Auto": # Get stats from indexer. @@ -170,25 +170,25 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory return { 'dataset_type': indexer.dataset_type, 'data': None } else: return stats - - # Stats provides features/base and resolution is bases/pixel, so + + # Stats provides features/base and resolution is bases/pixel, so # multiplying them yields features/pixel. features_per_pixel = stats[ 'data' ][ 'max' ] * float( kwargs[ 'resolution' ] ) - # Use heuristic based on features/pixel and region size to determine whether to + # Use heuristic based on features/pixel and region size to determine whether to # return coverage data. When zoomed out and region is large, features/pixel - # is determining factor. However, when sufficiently zoomed in and region is + # is determining factor. However, when sufficiently zoomed in and region is # small, coverage data is no longer provided. if int( high ) - int( low ) > 50000 and features_per_pixel > 1000: return indexer.get_data( chrom, low, high ) - + # # Provide individual data points. # - + # Get data provider. data_provider = data_provider_registry.get_data_provider( trans, original_dataset=dataset, source='data' ) - + # Allow max_vals top be data provider set if not passed if max_vals is None: max_vals = data_provider.get_default_max_vals() @@ -199,16 +199,16 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory data_dict = self.app.genomes.reference( trans, dbkey=dataset.dbkey, chrom=chrom, low=low, high=high ) if data_dict: ref_seq = data_dict[ 'data' ] - + # Get and return data from data_provider. - result = data_provider.get_data( chrom, int( low ), int( high ), int( start_val ), int( max_vals ), + result = data_provider.get_data( chrom, int( low ), int( high ), int( start_val ), int( max_vals ), ref_seq=ref_seq, **kwargs ) result.update( { 'dataset_type': data_provider.dataset_type, 'extra_info': extra_info } ) return result def _raw_data( self, trans, dataset, provider=None, **kwargs ): """ - Uses original (raw) dataset to return data. This method is useful + Uses original (raw) dataset to return data. This method is useful when the dataset is not yet indexed and hence using data would be slow because indexes need to be created. """ @@ -216,7 +216,7 @@ class DatasetsController( BaseAPIController, UsesVisualizationMixin, UsesHistory msg = self.check_dataset_state( trans, dataset ) if msg: return msg - + registry = trans.app.data_provider_registry # allow the caller to specifiy which provider is used diff --git a/lib/galaxy/webapps/galaxy/api/folder_contents.py b/lib/galaxy/webapps/galaxy/api/folder_contents.py index b5a1f02f62e..27e6ed199ec 100644 --- a/lib/galaxy/webapps/galaxy/api/folder_contents.py +++ b/lib/galaxy/webapps/galaxy/api/folder_contents.py @@ -53,7 +53,7 @@ class FolderContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrary parent_library = folder.parent_library except: folder = None - log.error( "FolderContentsController.index: Unable to retrieve folder %s" + log.error( "FolderContentsController.index: Unable to retrieve folder %s" % folder_id ) # TODO: Find the API's path to this folder if necessary. diff --git a/lib/galaxy/webapps/galaxy/api/folders.py b/lib/galaxy/webapps/galaxy/api/folders.py index 5cb3e71f19c..abb1e09d508 100644 --- a/lib/galaxy/webapps/galaxy/api/folders.py +++ b/lib/galaxy/webapps/galaxy/api/folders.py @@ -1,5 +1,5 @@ """ -API operations on folders +API operations on folders """ import logging, os, string, shutil, urllib, re, socket, traceback from cgi import escape, FieldStorage @@ -25,7 +25,7 @@ class FoldersController( BaseAPIController, UsesLibraryMixin, UsesLibraryMixinIt def show( self, trans, id, **kwd ): """ GET /api/folders/{encoded_folder_id} - Displays information about a folder + Displays information about a folder """ # Eliminate any 'F' in front of the folder id. Just take the # last 16 characters: @@ -33,7 +33,7 @@ class FoldersController( BaseAPIController, UsesLibraryMixin, UsesLibraryMixinIt id = id[-16:] # Retrieve the folder and return its contents encoded. Note that the # check_ownership=false since we are only displaying it. - content = self.get_library_folder( trans, id, check_ownership=False, + content = self.get_library_folder( trans, id, check_ownership=False, check_accessible=True ) return self.encode_all_ids( trans, content.dictify( view='element' ) ) @@ -42,7 +42,7 @@ class FoldersController( BaseAPIController, UsesLibraryMixin, UsesLibraryMixinIt """ POST /api/folders/{encoded_folder_id} Create a new object underneath the one specified in the parameters. - This will use the same parameters and semantics as + This will use the same parameters and semantics as /api/libraries/{LibID}/contents/{ContentId} for consistency. This means that datasets and folders can be generated. Note that /api/libraries/{LibID}/contents/{ContentId} did not need the library @@ -89,18 +89,18 @@ class FoldersController( BaseAPIController, UsesLibraryMixin, UsesLibraryMixinIt rval.append( dict( id = encoded_id, name = v.name, url = url_for( 'folder', id=encoded_id ) ) ) - else: + else: log.debug( "Error creating folder; setting output and status" ) trans.response.status = status - rval = output - return rval + rval = output + return rval @web.expose_api def update( self, trans, id, library_id, payload, **kwd ): """ PUT /api/folders/{encoded_folder_id} For now this does nothing. There are no semantics for folders that - indicates that an update operation is needed; the existing + indicates that an update operation is needed; the existing library_contents folder does not allow for update, either. """ pass diff --git a/lib/galaxy/webapps/galaxy/api/forms.py b/lib/galaxy/webapps/galaxy/api/forms.py index c0cbb4671e2..effcff8cdf6 100644 --- a/lib/galaxy/webapps/galaxy/api/forms.py +++ b/lib/galaxy/webapps/galaxy/api/forms.py @@ -10,7 +10,7 @@ from elementtree.ElementTree import XML log = logging.getLogger( __name__ ) class FormDefinitionAPIController( BaseAPIController ): - + @web.expose_api def index( self, trans, **kwd ): """ diff --git a/lib/galaxy/webapps/galaxy/api/genomes.py b/lib/galaxy/webapps/galaxy/api/genomes.py index 95fbde1db07..f330d26bec6 100644 --- a/lib/galaxy/webapps/galaxy/api/genomes.py +++ b/lib/galaxy/webapps/galaxy/api/genomes.py @@ -13,20 +13,20 @@ class GenomesController( BaseAPIController ): """ RESTful controller for interactions with genome data. """ - + @web.expose_api def index( self, trans, **kwd ): """ GET /api/genomes: returns a list of installed genomes - """ - + """ + return self.app.genomes.get_dbkeys( trans, **kwd ) @web.json def show( self, trans, id, num=None, chrom=None, low=None, high=None, **kwd ): """ GET /api/genomes/{id} - + Returns information about build """ @@ -41,13 +41,13 @@ class GenomesController( BaseAPIController ): else: rval = self.app.genomes.chroms( trans, dbkey=id, num=num, chrom=chrom, low=low ) return rval - + @web.expose_api def create( self, trans, payload, **kwd ): """ POST /api/genomes Download and/or index a genome. - + Parameters:: liftover None or array of liftover url partial paths dbkey DB key of the build to download @@ -59,10 +59,10 @@ class GenomesController( BaseAPIController ): Returns:: If no error: dict( status: 'ok', job: ) - + If error: dict( status: 'error', error: ) - + """ #??: Planned? #Parameters:: diff --git a/lib/galaxy/webapps/galaxy/api/group_roles.py b/lib/galaxy/webapps/galaxy/api/group_roles.py index d898cf8e6fd..4c710cb7799 100644 --- a/lib/galaxy/webapps/galaxy/api/group_roles.py +++ b/lib/galaxy/webapps/galaxy/api/group_roles.py @@ -1,124 +1,124 @@ -""" -API operations on Group objects. -""" -import logging -from galaxy.web.base.controller import BaseAPIController, url_for -from galaxy import web - -log = logging.getLogger( __name__ ) - -class GroupRolesAPIController( BaseAPIController ): - - @web.expose_api - @web.require_admin - def index( self, trans, group_id, **kwd ): - """ - GET /api/groups/{encoded_group_id}/roles - Displays a collection (list) of groups. - """ - decoded_group_id = trans.security.decode_id( group_id ) - try: - group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) - except: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str( group_id ) - rval = [] - try: - for gra in group.roles: - role = gra.role - encoded_id = trans.security.encode_id( role.id ) - rval.append( dict( id = encoded_id, - name = role.name, - url = url_for( 'group_role', group_id=group_id, id=encoded_id, ) ) ) - except Exception, e: - rval = "Error in group API at listing roles" - log.error( rval + ": %s" % str(e) ) - trans.response.status = 500 - return rval - - @web.expose_api - @web.require_admin - def show( self, trans, id, group_id, **kwd ): - """ - GET /api/groups/{encoded_group_id}/roles/{encoded_role_id} - Displays information about a group role. - """ - role_id = id - decoded_group_id = trans.security.decode_id( group_id ) - decoded_role_id = trans.security.decode_id( role_id ) - item = None - try: - group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) - role = trans.sa_session.query( trans.app.model.Role ).get( decoded_role_id ) - for gra in group.roles: - if gra.role == role: - item = dict( id = role_id, - name = role.name, - url = url_for( 'group_role', group_id=group_id, id=role_id) ) # TODO Fix This - if not item: - item = "role %s not in group %s" % (role.name,group.name) - except Exception, e: - item = "Error in group_role API group %s role %s" % (group.name, role.name) - log.error(item + ": %s" % str(e)) - return item - - @web.expose_api - @web.require_admin - def update( self, trans, id, group_id, **kwd ): - """ - PUT /api/groups/{encoded_group_id}/roles/{encoded_role_id} - Adds a role to a group - """ - role_id = id - decoded_group_id = trans.security.decode_id( group_id ) - decoded_role_id = trans.security.decode_id( role_id ) - item = None - try: - group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) - role = trans.sa_session.query( trans.app.model.Role ).get( decoded_role_id ) - for gra in group.roles: - if gra.role == role: - item = dict( id = role_id, - name = role.name, - url = url_for( 'group_role', group_id=group_id, id=role_id) ) - if not item: - gra = trans.app.model.GroupRoleAssociation( group, role ) - # Add GroupRoleAssociation - trans.sa_session.add( gra ) - trans.sa_session.flush() - item = dict( id = role_id, - name = role.name, - url = url_for( 'group_role', group_id=group_id, id=role_id) ) - except Exception, e: - item = "Error in group_role API Adding role %s to group %s" % (role.name,group.name) - log.error(item + ": %s" % str(e)) - return item - - @web.expose_api - @web.require_admin - def delete( self, trans, id, group_id, **kwd ): - """ - DELETE /api/groups/{encoded_group_id}/roles/{encoded_role_id} - Removes a role from a group - """ - role_id = id - decoded_group_id = trans.security.decode_id( group_id ) - decoded_role_id = trans.security.decode_id( role_id ) - try: - group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) - role = trans.sa_session.query( trans.app.model.Role ).get( decoded_role_id ) - for gra in group.roles: - if gra.role == role: - trans.sa_session.delete( gra ) - trans.sa_session.flush() - item = dict( id = role_id, - name = role.name, - url = url_for( 'group_role', group_id=group_id, id=role_id) ) - if not item: - item = "role %s not in group %s" % (role.name,group.name) - except Exception, e: - item = "Error in group_role API Removing role %s from group %s" % (role.name,group.name) - log.error(item + ": %s" % str(e)) - return item +""" +API operations on Group objects. +""" +import logging +from galaxy.web.base.controller import BaseAPIController, url_for +from galaxy import web + +log = logging.getLogger( __name__ ) + +class GroupRolesAPIController( BaseAPIController ): + + @web.expose_api + @web.require_admin + def index( self, trans, group_id, **kwd ): + """ + GET /api/groups/{encoded_group_id}/roles + Displays a collection (list) of groups. + """ + decoded_group_id = trans.security.decode_id( group_id ) + try: + group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) + except: + group = None + if not group: + trans.response.status = 400 + return "Invalid group id ( %s ) specified." % str( group_id ) + rval = [] + try: + for gra in group.roles: + role = gra.role + encoded_id = trans.security.encode_id( role.id ) + rval.append( dict( id = encoded_id, + name = role.name, + url = url_for( 'group_role', group_id=group_id, id=encoded_id, ) ) ) + except Exception, e: + rval = "Error in group API at listing roles" + log.error( rval + ": %s" % str(e) ) + trans.response.status = 500 + return rval + + @web.expose_api + @web.require_admin + def show( self, trans, id, group_id, **kwd ): + """ + GET /api/groups/{encoded_group_id}/roles/{encoded_role_id} + Displays information about a group role. + """ + role_id = id + decoded_group_id = trans.security.decode_id( group_id ) + decoded_role_id = trans.security.decode_id( role_id ) + item = None + try: + group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) + role = trans.sa_session.query( trans.app.model.Role ).get( decoded_role_id ) + for gra in group.roles: + if gra.role == role: + item = dict( id = role_id, + name = role.name, + url = url_for( 'group_role', group_id=group_id, id=role_id) ) # TODO Fix This + if not item: + item = "role %s not in group %s" % (role.name,group.name) + except Exception, e: + item = "Error in group_role API group %s role %s" % (group.name, role.name) + log.error(item + ": %s" % str(e)) + return item + + @web.expose_api + @web.require_admin + def update( self, trans, id, group_id, **kwd ): + """ + PUT /api/groups/{encoded_group_id}/roles/{encoded_role_id} + Adds a role to a group + """ + role_id = id + decoded_group_id = trans.security.decode_id( group_id ) + decoded_role_id = trans.security.decode_id( role_id ) + item = None + try: + group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) + role = trans.sa_session.query( trans.app.model.Role ).get( decoded_role_id ) + for gra in group.roles: + if gra.role == role: + item = dict( id = role_id, + name = role.name, + url = url_for( 'group_role', group_id=group_id, id=role_id) ) + if not item: + gra = trans.app.model.GroupRoleAssociation( group, role ) + # Add GroupRoleAssociation + trans.sa_session.add( gra ) + trans.sa_session.flush() + item = dict( id = role_id, + name = role.name, + url = url_for( 'group_role', group_id=group_id, id=role_id) ) + except Exception, e: + item = "Error in group_role API Adding role %s to group %s" % (role.name,group.name) + log.error(item + ": %s" % str(e)) + return item + + @web.expose_api + @web.require_admin + def delete( self, trans, id, group_id, **kwd ): + """ + DELETE /api/groups/{encoded_group_id}/roles/{encoded_role_id} + Removes a role from a group + """ + role_id = id + decoded_group_id = trans.security.decode_id( group_id ) + decoded_role_id = trans.security.decode_id( role_id ) + try: + group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) + role = trans.sa_session.query( trans.app.model.Role ).get( decoded_role_id ) + for gra in group.roles: + if gra.role == role: + trans.sa_session.delete( gra ) + trans.sa_session.flush() + item = dict( id = role_id, + name = role.name, + url = url_for( 'group_role', group_id=group_id, id=role_id) ) + if not item: + item = "role %s not in group %s" % (role.name,group.name) + except Exception, e: + item = "Error in group_role API Removing role %s from group %s" % (role.name,group.name) + log.error(item + ": %s" % str(e)) + return item diff --git a/lib/galaxy/webapps/galaxy/api/group_users.py b/lib/galaxy/webapps/galaxy/api/group_users.py index 5d95f60f799..d6f2d0234df 100644 --- a/lib/galaxy/webapps/galaxy/api/group_users.py +++ b/lib/galaxy/webapps/galaxy/api/group_users.py @@ -1,124 +1,124 @@ -""" -API operations on Group objects. -""" -import logging -from galaxy.web.base.controller import BaseAPIController, url_for -from galaxy import web - -log = logging.getLogger( __name__ ) - -class GroupUsersAPIController( BaseAPIController ): - - @web.expose_api - @web.require_admin - def index( self, trans, group_id, **kwd ): - """ - GET /api/groups/{encoded_group_id}/users - Displays a collection (list) of groups. - """ - decoded_group_id = trans.security.decode_id( group_id ) - try: - group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) - except: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str( group_id ) - rval = [] - try: - for uga in group.users: - user = uga.user - encoded_id = trans.security.encode_id( user.id ) - rval.append( dict( id = encoded_id, - email = user.email, - url = url_for( 'group_user', group_id=group_id, id=encoded_id, ) ) ) - except Exception, e: - rval = "Error in group API at listing users" - log.error( rval + ": %s" % str(e) ) - trans.response.status = 500 - return rval - - @web.expose_api - @web.require_admin - def show( self, trans, id, group_id, **kwd ): - """ - GET /api/groups/{encoded_group_id}/users/{encoded_user_id} - Displays information about a group user. - """ - user_id = id - decoded_group_id = trans.security.decode_id( group_id ) - decoded_user_id = trans.security.decode_id( user_id ) - item = None - try: - group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) - user = trans.sa_session.query( trans.app.model.User ).get( decoded_user_id ) - for uga in group.users: - if uga.user == user: - item = dict( id = user_id, - email = user.email, - url = url_for( 'group_user', group_id=group_id, id=user_id) ) # TODO Fix This - if not item: - item = "user %s not in group %s" % (user.email,group.name) - except Exception, e: - item = "Error in group_user API group %s user %s" % (group.name, user.email) - log.error(item + ": %s" % str(e)) - return item - - @web.expose_api - @web.require_admin - def update( self, trans, id, group_id, **kwd ): - """ - PUT /api/groups/{encoded_group_id}/users/{encoded_user_id} - Adds a user to a group - """ - user_id = id - decoded_group_id = trans.security.decode_id( group_id ) - decoded_user_id = trans.security.decode_id( user_id ) - item = None - try: - group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) - user = trans.sa_session.query( trans.app.model.User ).get( decoded_user_id ) - for uga in group.users: - if uga.user == user: - item = dict( id = user_id, - email = user.email, - url = url_for( 'group_user', group_id=group_id, id=user_id) ) - if not item: - uga = trans.app.model.UserGroupAssociation( user, group ) - # Add UserGroupAssociations - trans.sa_session.add( uga ) - trans.sa_session.flush() - item = dict( id = user_id, - email = user.email, - url = url_for( 'group_user', group_id=group_id, id=user_id) ) - except Exception, e: - item = "Error in group_user API Adding user %s to group %s" % (user.email,group.name) - log.error(item + ": %s" % str(e)) - return item - - @web.expose_api - @web.require_admin - def delete( self, trans, id, group_id, **kwd ): - """ - DELETE /api/groups/{encoded_group_id}/users/{encoded_user_id} - Removes a user from a group - """ - user_id = id - decoded_group_id = trans.security.decode_id( group_id ) - decoded_user_id = trans.security.decode_id( user_id ) - try: - group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) - user = trans.sa_session.query( trans.app.model.User ).get( decoded_user_id ) - for uga in group.users: - if uga.user == user: - trans.sa_session.delete( uga ) - trans.sa_session.flush() - item = dict( id = user_id, - email = user.email, - url = url_for( 'group_user', group_id=group_id, id=user_id) ) - if not item: - item = "user %s not in group %s" % (user.email,group.name) - except Exception, e: - item = "Error in group_user API Removing user %s from group %s" % (user.email,group.name) - log.error(item + ": %s" % str(e)) - return item +""" +API operations on Group objects. +""" +import logging +from galaxy.web.base.controller import BaseAPIController, url_for +from galaxy import web + +log = logging.getLogger( __name__ ) + +class GroupUsersAPIController( BaseAPIController ): + + @web.expose_api + @web.require_admin + def index( self, trans, group_id, **kwd ): + """ + GET /api/groups/{encoded_group_id}/users + Displays a collection (list) of groups. + """ + decoded_group_id = trans.security.decode_id( group_id ) + try: + group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) + except: + group = None + if not group: + trans.response.status = 400 + return "Invalid group id ( %s ) specified." % str( group_id ) + rval = [] + try: + for uga in group.users: + user = uga.user + encoded_id = trans.security.encode_id( user.id ) + rval.append( dict( id = encoded_id, + email = user.email, + url = url_for( 'group_user', group_id=group_id, id=encoded_id, ) ) ) + except Exception, e: + rval = "Error in group API at listing users" + log.error( rval + ": %s" % str(e) ) + trans.response.status = 500 + return rval + + @web.expose_api + @web.require_admin + def show( self, trans, id, group_id, **kwd ): + """ + GET /api/groups/{encoded_group_id}/users/{encoded_user_id} + Displays information about a group user. + """ + user_id = id + decoded_group_id = trans.security.decode_id( group_id ) + decoded_user_id = trans.security.decode_id( user_id ) + item = None + try: + group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) + user = trans.sa_session.query( trans.app.model.User ).get( decoded_user_id ) + for uga in group.users: + if uga.user == user: + item = dict( id = user_id, + email = user.email, + url = url_for( 'group_user', group_id=group_id, id=user_id) ) # TODO Fix This + if not item: + item = "user %s not in group %s" % (user.email,group.name) + except Exception, e: + item = "Error in group_user API group %s user %s" % (group.name, user.email) + log.error(item + ": %s" % str(e)) + return item + + @web.expose_api + @web.require_admin + def update( self, trans, id, group_id, **kwd ): + """ + PUT /api/groups/{encoded_group_id}/users/{encoded_user_id} + Adds a user to a group + """ + user_id = id + decoded_group_id = trans.security.decode_id( group_id ) + decoded_user_id = trans.security.decode_id( user_id ) + item = None + try: + group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) + user = trans.sa_session.query( trans.app.model.User ).get( decoded_user_id ) + for uga in group.users: + if uga.user == user: + item = dict( id = user_id, + email = user.email, + url = url_for( 'group_user', group_id=group_id, id=user_id) ) + if not item: + uga = trans.app.model.UserGroupAssociation( user, group ) + # Add UserGroupAssociations + trans.sa_session.add( uga ) + trans.sa_session.flush() + item = dict( id = user_id, + email = user.email, + url = url_for( 'group_user', group_id=group_id, id=user_id) ) + except Exception, e: + item = "Error in group_user API Adding user %s to group %s" % (user.email,group.name) + log.error(item + ": %s" % str(e)) + return item + + @web.expose_api + @web.require_admin + def delete( self, trans, id, group_id, **kwd ): + """ + DELETE /api/groups/{encoded_group_id}/users/{encoded_user_id} + Removes a user from a group + """ + user_id = id + decoded_group_id = trans.security.decode_id( group_id ) + decoded_user_id = trans.security.decode_id( user_id ) + try: + group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) + user = trans.sa_session.query( trans.app.model.User ).get( decoded_user_id ) + for uga in group.users: + if uga.user == user: + trans.sa_session.delete( uga ) + trans.sa_session.flush() + item = dict( id = user_id, + email = user.email, + url = url_for( 'group_user', group_id=group_id, id=user_id) ) + if not item: + item = "user %s not in group %s" % (user.email,group.name) + except Exception, e: + item = "Error in group_user API Removing user %s from group %s" % (user.email,group.name) + log.error(item + ": %s" % str(e)) + return item diff --git a/lib/galaxy/webapps/galaxy/api/groups.py b/lib/galaxy/webapps/galaxy/api/groups.py index 9d2332b1c62..bb804cf07bd 100644 --- a/lib/galaxy/webapps/galaxy/api/groups.py +++ b/lib/galaxy/webapps/galaxy/api/groups.py @@ -1,128 +1,128 @@ -""" -API operations on Group objects. -""" -import logging -from galaxy.web.base.controller import BaseAPIController, url_for -from galaxy import web - - -log = logging.getLogger( __name__ ) - - -class GroupAPIController( BaseAPIController ): - - @web.expose_api - @web.require_admin - def index( self, trans, **kwd ): - """ - GET /api/groups - Displays a collection (list) of groups. - """ - rval = [] - for group in trans.sa_session.query( trans.app.model.Group ).filter( trans.app.model.Group.table.c.deleted == False ): - if trans.user_is_admin(): - item = group.dictify( value_mapper={ 'id': trans.security.encode_id } ) - encoded_id = trans.security.encode_id( group.id ) - item['url'] = url_for( 'group', id=encoded_id ) - rval.append( item ) - return rval - - @web.expose_api - def create( self, trans, payload, **kwd ): - """ - POST /api/groups - Creates a new group. - """ - log.info("groups payload%s\n" % (payload)) - if not trans.user_is_admin(): - trans.response.status = 403 - return "You are not authorized to create a new group." - name = payload.get( 'name', None ) - if not name: - trans.response.status = 400 - return "Enter a valid name" - if trans.sa_session.query( trans.app.model.Group ).filter( trans.app.model.Group.table.c.name==name ).first(): - trans.response.status = 400 - return "A group with that name already exists" - - group = trans.app.model.Group( name=name ) - trans.sa_session.add( group ) - user_ids = payload.get( 'user_ids', [] ) - for i in user_ids: - log.info("user_id: %s\n" % (i )) - log.info("%s %s\n" % (i, trans.security.decode_id( i ) )) - users = [ trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( i ) ) for i in user_ids ] - role_ids = payload.get( 'role_ids', [] ) - roles = [ trans.sa_session.query( trans.model.Role ).get( trans.security.decode_id( i ) ) for i in role_ids ] - trans.app.security_agent.set_entity_group_associations( groups=[ group ], roles=roles, users=users ) - """ - # Create the UserGroupAssociations - for user in users: - trans.app.security_agent.associate_user_group( user, group ) - # Create the GroupRoleAssociations - for role in roles: - trans.app.security_agent.associate_group_role( group, role ) - """ - trans.sa_session.flush() - encoded_id = trans.security.encode_id( group.id ) - item = group.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) - item['url'] = url_for( 'group', id=encoded_id ) - return [ item ] - - @web.expose_api - @web.require_admin - def show( self, trans, id, **kwd ): - """ - GET /api/groups/{encoded_group_id} - Displays information about a group. - """ - group_id = id - try: - decoded_group_id = trans.security.decode_id( group_id ) - except TypeError: - trans.response.status = 400 - return "Malformed group id ( %s ) specified, unable to decode." % str( group_id ) - try: - group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) - except: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str( group_id ) - item = group.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) - item['url'] = url_for( 'group', id=group_id ) - item['users_url'] = url_for( 'group_users', group_id=group_id ) - item['roles_url'] = url_for( 'group_roles', group_id=group_id ) - return item - - @web.expose_api - @web.require_admin - def update( self, trans, id, payload, **kwd ): - """ - PUT /api/groups/{encoded_group_id} - Modifies a group. - """ - group_id = id - try: - decoded_group_id = trans.security.decode_id( group_id ) - except TypeError: - trans.response.status = 400 - return "Malformed group id ( %s ) specified, unable to decode." % str( group_id ) - try: - group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) - except: - group = None - if not group: - trans.response.status = 400 - return "Invalid group id ( %s ) specified." % str( group_id ) - name = payload.get( 'name', None ) - if name: - group.name = name - trans.sa_session.add(group) - user_ids = payload.get( 'user_ids', [] ) - users = [ trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( i ) ) for i in user_ids ] - role_ids = payload.get( 'role_ids', [] ) - roles = [ trans.sa_session.query( trans.model.Role ).get( trans.security.decode_id( i ) ) for i in role_ids ] - trans.app.security_agent.set_entity_group_associations( groups=[ group ], roles=roles, users=users,delete_existing_assocs=False ) - trans.sa_session.flush() - +""" +API operations on Group objects. +""" +import logging +from galaxy.web.base.controller import BaseAPIController, url_for +from galaxy import web + + +log = logging.getLogger( __name__ ) + + +class GroupAPIController( BaseAPIController ): + + @web.expose_api + @web.require_admin + def index( self, trans, **kwd ): + """ + GET /api/groups + Displays a collection (list) of groups. + """ + rval = [] + for group in trans.sa_session.query( trans.app.model.Group ).filter( trans.app.model.Group.table.c.deleted == False ): + if trans.user_is_admin(): + item = group.dictify( value_mapper={ 'id': trans.security.encode_id } ) + encoded_id = trans.security.encode_id( group.id ) + item['url'] = url_for( 'group', id=encoded_id ) + rval.append( item ) + return rval + + @web.expose_api + def create( self, trans, payload, **kwd ): + """ + POST /api/groups + Creates a new group. + """ + log.info("groups payload%s\n" % (payload)) + if not trans.user_is_admin(): + trans.response.status = 403 + return "You are not authorized to create a new group." + name = payload.get( 'name', None ) + if not name: + trans.response.status = 400 + return "Enter a valid name" + if trans.sa_session.query( trans.app.model.Group ).filter( trans.app.model.Group.table.c.name==name ).first(): + trans.response.status = 400 + return "A group with that name already exists" + + group = trans.app.model.Group( name=name ) + trans.sa_session.add( group ) + user_ids = payload.get( 'user_ids', [] ) + for i in user_ids: + log.info("user_id: %s\n" % (i )) + log.info("%s %s\n" % (i, trans.security.decode_id( i ) )) + users = [ trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( i ) ) for i in user_ids ] + role_ids = payload.get( 'role_ids', [] ) + roles = [ trans.sa_session.query( trans.model.Role ).get( trans.security.decode_id( i ) ) for i in role_ids ] + trans.app.security_agent.set_entity_group_associations( groups=[ group ], roles=roles, users=users ) + """ + # Create the UserGroupAssociations + for user in users: + trans.app.security_agent.associate_user_group( user, group ) + # Create the GroupRoleAssociations + for role in roles: + trans.app.security_agent.associate_group_role( group, role ) + """ + trans.sa_session.flush() + encoded_id = trans.security.encode_id( group.id ) + item = group.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) + item['url'] = url_for( 'group', id=encoded_id ) + return [ item ] + + @web.expose_api + @web.require_admin + def show( self, trans, id, **kwd ): + """ + GET /api/groups/{encoded_group_id} + Displays information about a group. + """ + group_id = id + try: + decoded_group_id = trans.security.decode_id( group_id ) + except TypeError: + trans.response.status = 400 + return "Malformed group id ( %s ) specified, unable to decode." % str( group_id ) + try: + group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) + except: + group = None + if not group: + trans.response.status = 400 + return "Invalid group id ( %s ) specified." % str( group_id ) + item = group.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) + item['url'] = url_for( 'group', id=group_id ) + item['users_url'] = url_for( 'group_users', group_id=group_id ) + item['roles_url'] = url_for( 'group_roles', group_id=group_id ) + return item + + @web.expose_api + @web.require_admin + def update( self, trans, id, payload, **kwd ): + """ + PUT /api/groups/{encoded_group_id} + Modifies a group. + """ + group_id = id + try: + decoded_group_id = trans.security.decode_id( group_id ) + except TypeError: + trans.response.status = 400 + return "Malformed group id ( %s ) specified, unable to decode." % str( group_id ) + try: + group = trans.sa_session.query( trans.app.model.Group ).get( decoded_group_id ) + except: + group = None + if not group: + trans.response.status = 400 + return "Invalid group id ( %s ) specified." % str( group_id ) + name = payload.get( 'name', None ) + if name: + group.name = name + trans.sa_session.add(group) + user_ids = payload.get( 'user_ids', [] ) + users = [ trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( i ) ) for i in user_ids ] + role_ids = payload.get( 'role_ids', [] ) + roles = [ trans.sa_session.query( trans.model.Role ).get( trans.security.decode_id( i ) ) for i in role_ids ] + trans.app.security_agent.set_entity_group_associations( groups=[ group ], roles=roles, users=users,delete_existing_assocs=False ) + trans.sa_session.flush() + diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 676a602eb59..4a4a0076d5f 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -128,7 +128,7 @@ class HistoriesController( BaseAPIController, UsesHistoryMixin ): :type payload: dict :param payload: (optional) dictionary structure containing: * name: the new history's name - + :rtype: dict :returns: element view of new history """ @@ -159,7 +159,7 @@ class HistoriesController( BaseAPIController, UsesHistoryMixin ): :param id: the encoded id of the history to delete :type kwd: dict :param kwd: (optional) dictionary structure containing: - + * payload: a dictionary itself containing: * purge: if True, purge the history and all of it's HDAs @@ -255,7 +255,7 @@ class HistoriesController( BaseAPIController, UsesHistoryMixin ): :type payload: dict :param payload: a dictionary containing any or all the fields in :func:`galaxy.model.History.dictify` and/or the following: - + * annotation: an annotation for the history :rtype: dict diff --git a/lib/galaxy/webapps/galaxy/api/libraries.py b/lib/galaxy/webapps/galaxy/api/libraries.py index 399783131b7..56ef2873b28 100644 --- a/lib/galaxy/webapps/galaxy/api/libraries.py +++ b/lib/galaxy/webapps/galaxy/api/libraries.py @@ -11,7 +11,7 @@ from paste.httpexceptions import HTTPBadRequest, HTTPForbidden log = logging.getLogger( __name__ ) class LibrariesController( BaseAPIController ): - + @web.expose_api def index( self, trans, deleted='False', **kwd ): """ diff --git a/lib/galaxy/webapps/galaxy/api/library_contents.py b/lib/galaxy/webapps/galaxy/api/library_contents.py index b3599795e55..7527069f62a 100644 --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -129,7 +129,7 @@ class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar :param library_id: encoded id string of the library that contains this item :type payload: dict :param payload: dictionary structure containing: - + * folder_id: the parent folder of the new item * create_type: the type of item to create ('file' or 'folder') * from_hda_id: (optional) the id of an accessible HDA to copy into the @@ -172,7 +172,7 @@ class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar if create_type == 'file' and from_hda_id: return self._copy_hda_to_library_folder( trans, from_hda_id, library_id, real_folder_id, ldda_message ) - #check for extended metadata, store it and pop it out of the param + #check for extended metadata, store it and pop it out of the param #otherwise sanitize_param will have a fit ex_meta_payload = None if 'extended_metadata' in payload: diff --git a/lib/galaxy/webapps/galaxy/api/permissions.py b/lib/galaxy/webapps/galaxy/api/permissions.py index bed7da8ebb7..7b0ebdc91e1 100644 --- a/lib/galaxy/webapps/galaxy/api/permissions.py +++ b/lib/galaxy/webapps/galaxy/api/permissions.py @@ -12,10 +12,10 @@ from galaxy.model.orm import * log = logging.getLogger( __name__ ) class PermissionsController( BaseAPIController ): - + # Method not ideally named @web.expose_api - def create( self, trans, library_id, payload, **kwd ): + def create( self, trans, library_id, payload, **kwd ): """ POST /api/libraries/{encoded_library_id}/permissions Updates the library permissions. diff --git a/lib/galaxy/webapps/galaxy/api/quotas.py b/lib/galaxy/webapps/galaxy/api/quotas.py index c6db3eb1313..a6fe9c61ddd 100644 --- a/lib/galaxy/webapps/galaxy/api/quotas.py +++ b/lib/galaxy/webapps/galaxy/api/quotas.py @@ -50,7 +50,7 @@ class QuotaAPIController( BaseAPIController, Admin, AdminActions, UsesQuotaMixin """ quota = self.get_quota( trans, id, deleted=util.string_as_bool( deleted ) ) return quota.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) - + @web.expose_api @web.require_admin def create( self, trans, payload, **kwd ): diff --git a/lib/galaxy/webapps/galaxy/api/request_types.py b/lib/galaxy/webapps/galaxy/api/request_types.py index 7537b4b6a65..20fa14ca44c 100644 --- a/lib/galaxy/webapps/galaxy/api/request_types.py +++ b/lib/galaxy/webapps/galaxy/api/request_types.py @@ -92,7 +92,7 @@ class RequestTypeAPIController( BaseAPIController ): permissions[ trans.app.security_agent.get_action( v.action ) ] = roles if permissions: trans.app.security_agent.set_request_type_permissions( request_type, permissions ) - + #flush objects trans.sa_session.add( request_type ) trans.sa_session.flush() diff --git a/lib/galaxy/webapps/galaxy/api/requests.py b/lib/galaxy/webapps/galaxy/api/requests.py index 32d9cc2933d..11aa3e3189a 100644 --- a/lib/galaxy/webapps/galaxy/api/requests.py +++ b/lib/galaxy/webapps/galaxy/api/requests.py @@ -91,12 +91,12 @@ class RequestsAPIController( BaseAPIController ): trans.response.status = 400 return "Invalid request id ( %s ) specified." % str( request_id ) # check update type - if update_type == 'request_state': + if update_type == 'request_state': return self.__update_request_state( trans, encoded_request_id=id ) def __update_request_state( self, trans, encoded_request_id ): requests_common_cntrller = trans.webapp.controllers['requests_common'] - status, output = requests_common_cntrller.update_request_state( trans, - cntrller='api', + status, output = requests_common_cntrller.update_request_state( trans, + cntrller='api', request_id=encoded_request_id ) return status, output diff --git a/lib/galaxy/webapps/galaxy/api/roles.py b/lib/galaxy/webapps/galaxy/api/roles.py index c50840cb2ec..2429f0aac1f 100644 --- a/lib/galaxy/webapps/galaxy/api/roles.py +++ b/lib/galaxy/webapps/galaxy/api/roles.py @@ -46,7 +46,7 @@ class RoleAPIController( BaseAPIController ): item = role.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) item['url'] = url_for( 'role', id=role_id ) return item - + @web.expose_api def create( self, trans, payload, **kwd ): """ @@ -64,21 +64,21 @@ class RoleAPIController( BaseAPIController ): if trans.sa_session.query( trans.app.model.Role ).filter( trans.app.model.Role.table.c.name==name ).first(): trans.response.status = 400 return "A role with that name already exists" - + role_type = trans.app.model.Role.types.ADMIN #TODO: allow non-admins to create roles - + role = trans.app.model.Role( name=name, description=description, type=role_type ) trans.sa_session.add( role ) user_ids = payload.get( 'user_ids', [] ) users = [ trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( i ) ) for i in user_ids ] group_ids = payload.get( 'group_ids', [] ) - groups = [ trans.sa_session.query( trans.model.Group ).get( trans.security.decode_id( i ) ) for i in group_ids ] + groups = [ trans.sa_session.query( trans.model.Group ).get( trans.security.decode_id( i ) ) for i in group_ids ] # Create the UserRoleAssociations for user in users: - trans.app.security_agent.associate_user_role( user, role ) + trans.app.security_agent.associate_user_role( user, role ) # Create the GroupRoleAssociations for group in groups: - trans.app.security_agent.associate_group_role( group, role ) + trans.app.security_agent.associate_group_role( group, role ) trans.sa_session.flush() encoded_id = trans.security.encode_id( role.id ) item = role.dictify( view='element', value_mapper={ 'id': trans.security.encode_id } ) diff --git a/lib/galaxy/webapps/galaxy/api/samples.py b/lib/galaxy/webapps/galaxy/api/samples.py index d74f1d8e672..25b68145d3e 100644 --- a/lib/galaxy/webapps/galaxy/api/samples.py +++ b/lib/galaxy/webapps/galaxy/api/samples.py @@ -36,8 +36,8 @@ class SamplesAPIController( BaseAPIController ): rval = [] for sample in request.samples: item = sample.dictify() - item['url'] = url_for( 'samples', - request_id=trans.security.encode_id( request_id ), + item['url'] = url_for( 'samples', + request_id=trans.security.encode_id( request_id ), id=trans.security.encode_id( sample.id ) ) item['id'] = trans.security.encode_id( item['id'] ) rval.append( item ) @@ -88,9 +88,9 @@ class SamplesAPIController( BaseAPIController ): sample_id=sample_id, **payload ) return status, output - elif update_type == 'sample_state': + elif update_type == 'sample_state': return self.__update_sample_state( trans, sample, sample_id, **payload ) - elif update_type == 'sample_dataset_transfer_status': + elif update_type == 'sample_dataset_transfer_status': # update sample_dataset transfer status return self.__update_sample_dataset_status( trans, **payload ) @@ -114,8 +114,8 @@ class SamplesAPIController( BaseAPIController ): trans.response.status = 400 return "Invalid sample state requested ( %s )." % new_state_name requests_common_cntrller = trans.webapp.controllers[ 'requests_common' ] - status, output = requests_common_cntrller.update_sample_state( trans=trans, - cntrller='api', + status, output = requests_common_cntrller.update_sample_state( trans=trans, + cntrller='api', sample_ids=[ encoded_sample_id ], new_state=new_state, comment=comment ) @@ -132,8 +132,8 @@ class SamplesAPIController( BaseAPIController ): new_status = payload.pop( 'new_status' ) error_msg = payload.get( 'error_msg', '' ) requests_admin_cntrller = trans.webapp.controllers[ 'requests_admin' ] - status, output = requests_admin_cntrller.update_sample_dataset_status( trans=trans, - cntrller='api', + status, output = requests_admin_cntrller.update_sample_dataset_status( trans=trans, + cntrller='api', sample_dataset_ids=sample_dataset_ids, new_status=new_status, error_msg=error_msg ) diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index b22d2ec9932..8c75736fc2f 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -84,9 +84,9 @@ class ToolShedRepositoriesController( BaseAPIController ): """ POST /api/tool_shed_repositories/install_repository_revision Install a specified repository revision from a specified tool shed into Galaxy. - + :param key: the current Galaxy admin user's API key - + The following parameters are included in the payload. :param tool_shed_url (required): the base URL of the Tool Shed from which to install the Repository :param name (required): the name of the Repository @@ -279,12 +279,12 @@ class ToolShedRepositoriesController( BaseAPIController ): POST /api/tool_shed_repositories/install_repository_revisions Install one or more specified repository revisions from one or more specified tool sheds into Galaxy. The received parameters must be ordered lists so that positional values in tool_shed_urls, names, owners and changeset_revisions are associated. - + It's questionable whether this method is needed as the above method for installing a single repository can probably cover all desired scenarios. We'll keep this one around just in case... - + :param key: the current Galaxy admin user's API key - + The following parameters are included in the payload. :param tool_shed_urls: the base URLs of the Tool Sheds from which to install a specified Repository :param names: the names of the Repositories to be installed @@ -365,9 +365,9 @@ class ToolShedRepositoriesController( BaseAPIController ): """ POST /api/tool_shed_repositories/repair_repository_revision Repair a specified repository revision previously installed into Galaxy. - + :param key: the current Galaxy admin user's API key - + The following parameters are included in the payload. :param tool_shed_url (required): the base URL of the Tool Shed from which the Repository was installed :param name (required): the name of the Repository diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 99aef8c68aa..21299f725e3 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -11,7 +11,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): """ RESTful controller for interactions with tools. """ - + @web.expose_api def index( self, trans, **kwds ): """ @@ -19,17 +19,17 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): parameters: - in_panel - if true, tools are returned in panel structure, + in_panel - if true, tools are returned in panel structure, including sections and labels - trackster - if true, only tools that are compatible with + trackster - if true, only tools that are compatible with Trackster are returned """ - + # Read params. in_panel = util.string_as_bool( kwds.get( 'in_panel', 'True' ) ) trackster = util.string_as_bool( kwds.get( 'trackster', 'False' ) ) - + # Create return value. try: return self.app.toolbox.dictify( trans, in_panel=in_panel, trackster=trackster ) @@ -50,7 +50,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): log.error( 'could not convert tool (%s) to dictionary: %s', id, str( exc ), exc_info=True ) trans.response.status = 500 return { 'error': str( exc ) } - + @web.expose_api def create( self, trans, payload, **kwd ): """ @@ -61,9 +61,9 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): action = payload.get( 'action', None ) if action == 'rerun': return self._rerun_tool( trans, payload, **kwd ) - + # -- Execute tool. -- - + # Get tool. tool = trans.app.toolbox.get_tool( payload[ 'tool_id' ] ) if 'tool_id' in payload else None if not tool: @@ -79,7 +79,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): trans.security.decode_id(history_id)) else: target_history = None - + # Set up inputs. inputs = payload.get( 'inputs', {} ) # Find files coming in as multipart file data and add to inputs. @@ -108,11 +108,11 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): output_dict = output.dictify() outputs.append( trans.security.encode_dict_ids( output_dict ) ) return rval - + # # -- Helper methods -- # - + def _run_tool( self, trans, tool_id, target_dataset_id, **kwargs ): """ Run a tool. This method serves as a general purpose way to run tools asynchronously. @@ -120,7 +120,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): # # Set target history (the history that tool will use for outputs) using - # target dataset. If user owns dataset, put new data in original + # target dataset. If user owns dataset, put new data in original # dataset's history; if user does not own dataset (and hence is accessing # dataset via sharing), put new data in user's current history. # @@ -130,7 +130,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): else: target_history = trans.get_history( create=True ) - # HACK: tools require unencoded parameters but kwargs are typically + # HACK: tools require unencoded parameters but kwargs are typically # encoded, so try decoding all parameter values. for key, value in kwargs.items(): try: @@ -139,7 +139,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): except: pass - # + # # Execute tool. # tool = trans.app.toolbox.get_tool( tool_id ) @@ -155,16 +155,16 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): output_datasets = vars[ 'out_data' ].values() return self.add_track_async( trans, output_datasets[0].id ) - + def _rerun_tool( self, trans, payload, **kwargs ): """ - Rerun a tool to produce a new output dataset that corresponds to a + Rerun a tool to produce a new output dataset that corresponds to a dataset that a user is currently viewing. """ # # TODO: refactor to use same code as run_tool. - # + # # Run tool on region if region is specificied. run_on_regions = False @@ -180,8 +180,8 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): if len( regions ) > 1: # Sort by chrom name, start so that data is not fetched out of order. regions = sorted(regions, key=lambda r: (r.chrom.lower(), r.start)) - - # Merge overlapping regions so that regions do not overlap + + # Merge overlapping regions so that regions do not overlap # and hence data is not included multiple times. prev = regions[0] cur = regions[1] @@ -196,15 +196,15 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): prev = cur index += 1 - # Get next region or exit. + # Get next region or exit. if index == len( regions ): # Done. break else: cur = regions[ index ] - + run_on_regions = True - + # Dataset check. original_dataset = self.get_dataset( trans, payload[ 'target_dataset_id' ], check_ownership=False, check_accessible=True ) msg = self.check_dataset_state( trans, original_dataset ) @@ -221,9 +221,9 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): if not tool: return trans.app.model.Dataset.conversion_messages.NO_TOOL tool_params = dict( [ ( p.name, p.value ) for p in original_job.parameters ] ) - + # TODO: rather than set new inputs using dict of json'ed value, unpack parameters and set using set_param_value below. - # TODO: need to handle updates to conditional parameters; conditional + # TODO: need to handle updates to conditional parameters; conditional # params are stored in dicts (and dicts within dicts). new_inputs = payload[ 'inputs' ] tool_params.update( dict( [ ( key, to_json_string( value ) ) for key, value in new_inputs.items() if key in tool.inputs and new_inputs[ key ] is not None ] ) ) @@ -232,14 +232,14 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): # # If running tool on region, convert input datasets (create indices) so # that can regions of data can be quickly extracted. - # + # data_provider_registry = trans.app.data_provider_registry messages_list = [] if run_on_regions: for jida in original_job.input_datasets: input_dataset = jida.dataset data_provider = data_provider_registry.get_data_provider( trans, original_dataset=input_dataset, source='data' ) - if data_provider and ( not data_provider.converted_dataset + if data_provider and ( not data_provider.converted_dataset or data_provider.converted_dataset.state != trans.app.model.Dataset.states.OK ): # Can convert but no converted dataset yet, so return message about why. data_sources = input_dataset.datatype.data_sources @@ -254,8 +254,8 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): # # Set target history (the history that tool will use for inputs/outputs). - # If user owns dataset, put new data in original dataset's history; if - # user does not own dataset (and hence is accessing dataset via sharing), + # If user owns dataset, put new data in original dataset's history; if + # user does not own dataset (and hence is accessing dataset via sharing), # put new data in user's current history. # if original_dataset.history.user == trans.user: @@ -369,8 +369,8 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): # Set metadata. # TODO: set meta internally if dataset is small enough? - trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, - trans, incoming = { 'input1':new_dataset }, + trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, + trans, incoming = { 'input1':new_dataset }, overwrite=False, job_params={ "source" : "trackster" } ) # Add HDA subset association. subset_association = trans.app.model.HistoryDatasetAssociationSubset( hda=input_dataset, subset=new_dataset, location=regions_str ) @@ -384,12 +384,12 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): if not set_param_value( tool_params, jida.name, subset_dataset ): return { "error" : True, "message" : "error setting parameter %s" % jida.name } - # + # # Execute tool and handle outputs. # try: - subset_job, subset_job_outputs = tool.execute( trans, incoming=tool_params, - history=target_history, + subset_job, subset_job_outputs = tool.execute( trans, incoming=tool_params, + history=target_history, job_params={ "source" : "trackster" } ) except Exception, e: # Lots of things can go wrong when trying to execute tool. @@ -399,7 +399,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): output.visible = False trans.sa_session.flush() - # + # # Return new track that corresponds to the original dataset. # output_name = None @@ -410,7 +410,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): for joda in subset_job.output_datasets: if joda.name == output_name: output_dataset = joda.dataset - + dataset_dict = output_dataset.dictify() dataset_dict[ 'id' ] = trans.security.encode_id( dataset_dict[ 'id' ] ) dataset_dict[ 'track_config' ] = self.get_new_track_config( trans, output_dataset ); diff --git a/lib/galaxy/webapps/galaxy/api/visualizations.py b/lib/galaxy/webapps/galaxy/api/visualizations.py index 72d1a52f719..f53b1034ab7 100644 --- a/lib/galaxy/webapps/galaxy/api/visualizations.py +++ b/lib/galaxy/webapps/galaxy/api/visualizations.py @@ -24,11 +24,11 @@ class VisualizationsController( BaseAPIController, UsesVisualizationMixin, UsesA """ RESTful controller for interactions with visualizations. """ - + @web.expose_api def index( self, trans, **kwargs ): """ - GET /api/visualizations: + GET /api/visualizations: """ #TODO: search for vizsesses that apply to an object (sending model class and id? - how to do this?) rval = [] @@ -65,7 +65,7 @@ class VisualizationsController( BaseAPIController, UsesVisualizationMixin, UsesA log.exception( 'visualizations index failed: %s' %( str( exception ) ) ) return rval - + @web.json def show( self, trans, id, **kwargs ): """ @@ -104,7 +104,7 @@ class VisualizationsController( BaseAPIController, UsesVisualizationMixin, UsesA log.exception( 'visualization show failed (%s): %s' %( id, str( exception ) ) ) return rval - + @web.expose_api def create( self, trans, payload, **kwargs ): """ diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 81280cc4401..9cfa1e0ceb4 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -55,7 +55,7 @@ def app_factory( global_conf, **kwargs ): webapp.add_route( '/async/:tool_id/:data_id/:data_secret', controller='async', action='index', tool_id=None, data_id=None, data_secret=None ) webapp.add_route( '/:controller/:action', action='index' ) webapp.add_route( '/:action', controller='root', action='index' ) - + # allow for subdirectories in extra_files_path webapp.add_route( '/datasets/:dataset_id/display/{filename:.+?}', controller='dataset', action='display', dataset_id=None, filename=None) webapp.add_route( '/datasets/:dataset_id/:action/:filename', controller='dataset', action='index', dataset_id=None, filename=None) @@ -83,13 +83,13 @@ def app_factory( global_conf, **kwargs ): 'contents', controller='library_contents', name_prefix='library_', - path_prefix='/api/libraries/:library_id', + path_prefix='/api/libraries/:library_id', parent_resources=dict( member_name='library', collection_name='libraries' ) ) webapp.mapper.resource( 'content', 'contents', controller='history_contents', name_prefix='history_', - path_prefix='/api/histories/:history_id', + path_prefix='/api/histories/:history_id', parent_resources=dict( member_name='history', collection_name='histories' ) ) webapp.mapper.connect("history_contents_display", "/api/histories/:history_id/contents/:history_content_id/display", @@ -122,7 +122,7 @@ def app_factory( global_conf, **kwargs ): name_prefix="workflow_", path_prefix='/api/workflows/:workflow_id' ) - _add_item_extended_metadata_controller( webapp, + _add_item_extended_metadata_controller( webapp, name_prefix="library_dataset_", path_prefix='/api/libraries/:library_id/contents/:library_content_id' ) @@ -157,7 +157,7 @@ def app_factory( global_conf, **kwargs ): webapp.mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' ) webapp.mapper.resource( 'configuration', 'configuration', path_prefix='/api' ) #webapp.mapper.connect( 'run_workflow', '/api/workflow/{workflow_id}/library/{library_id}', controller='workflows', action='run', workflow_id=None, library_id=None, conditions=dict(method=["GET"]) ) - webapp.mapper.resource( 'search', 'search', path_prefix='/api' ) + webapp.mapper.resource( 'search', 'search', path_prefix='/api' ) # visualizations registry generic template renderer webapp.add_route( '/visualization/show/:visualization_name', @@ -259,7 +259,7 @@ def _add_item_provenance_controller( webapp, name_prefix, path_prefix, **kwd ): def wrap_in_middleware( app, global_conf, **local_conf ): """ - Based on the configuration wrap `app` in a set of common and useful + Based on the configuration wrap `app` in a set of common and useful middleware. """ # Merge the global and local configurations @@ -280,7 +280,7 @@ def wrap_in_middleware( app, global_conf, **local_conf ): display_servers = util.listify( conf.get( 'display_servers', '' ) ), admin_users = conf.get( 'admin_users', '' ).split( ',' ) ) log.debug( "Enabling 'remote user' middleware" ) - # The recursive middleware allows for including requests in other + # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): from paste import recursive @@ -334,7 +334,7 @@ def wrap_in_middleware( app, global_conf, **local_conf ): app = RequestIDMiddleware( app ) log.debug( "Enabling 'Request ID' middleware" ) return app - + def wrap_in_static( app, global_conf, plugin_frameworks=None, **local_conf ): from paste.urlmap import URLMap from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static @@ -367,7 +367,7 @@ def wrap_in_static( app, global_conf, plugin_frameworks=None, **local_conf ): # URL mapper becomes the root webapp return urlmap - + def build_template_error_formatters(): """ Build a list of template error formatters for WebError. When an error diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index 5ee1651269a..0ccce0549ad 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -81,8 +81,8 @@ class UserListGrid( grids.Grid ): # Columns that are valid for filtering but are not visible. grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -168,8 +168,8 @@ class RoleListGrid( grids.Grid ): # Columns that are valid for filtering but are not visible. grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1], columns[2] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -241,8 +241,8 @@ class GroupListGrid( grids.Grid ): # Columns that are valid for filtering but are not visible. grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1], columns[2] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -332,8 +332,8 @@ class QuotaListGrid( grids.Grid ): # Columns that are valid for filtering but are not visible. grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1], columns[2] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -415,7 +415,7 @@ class ToolVersionListGrid( grids.Grid ): attach_popup=False ), ToolVersionsColumn( "Version lineage by tool id (parent/child ordered)" ) ] - columns.append( grids.MulticolFilterColumn( "Search tool id", + columns.append( grids.MulticolFilterColumn( "Search tool id", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, @@ -431,7 +431,7 @@ class ToolVersionListGrid( grids.Grid ): return trans.sa_session.query( self.model_class ) class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaParamParser ): - + user_list_grid = UserListGrid() role_list_grid = RoleListGrid() group_list_grid = GroupListGrid() @@ -661,7 +661,7 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP status='error' ) ) if do_op == True or ( do_op != False and params.get( do_op, False ) ): try: - message = op_method( quota, params ) + message = op_method( quota, params ) return None, trans.response.send_redirect( web.url_for( controller='admin', action='quotas', webapp=params.webapp, diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index d32ed386f35..98f6c793e70 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -105,7 +105,7 @@ class AdminToolshed( AdminGalaxy ): # Since we're reinstalling the repository we need to find the latest changeset revision to which it can be updated so that we # can reset the metadata if necessary. This will ensure that information about repository dependencies and tool dependencies # will be current. Only allow selecting a different section in the tool panel if the repository was uninstalled and it contained - # tools that should be displayed in the tool panel. + # tools that should be displayed in the tool panel. changeset_revision_dict = repository_util.get_update_to_changeset_revision_and_ctx_rev( trans, repository ) current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None ) current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None ) @@ -321,7 +321,7 @@ class AdminToolshed( AdminGalaxy ): @web.require_admin def get_tool_dependencies( self, trans, repository_id, repository_name, repository_owner, changeset_revision ): """ - Send a request to the appropriate tool shed to retrieve the dictionary of tool dependencies defined for the received repository name, + Send a request to the appropriate tool shed to retrieve the dictionary of tool dependencies defined for the received repository name, owner and changeset revision. The received repository_id is the encoded id of the installed tool shed repository in Galaxy. We need it so that we can derive the tool shed from which it was installed. """ @@ -394,7 +394,7 @@ class AdminToolshed( AdminGalaxy ): repository_id=repository_id, changeset_revision=changeset_revision ) # Save the workflow in the Galaxy database. - # Pass workflow_dict along to create annotation at this point + # Pass workflow_dict along to create annotation at this point stored_workflow = workflow_util.save_workflow( trans, workflow, workflow_dict ) # Use the latest version of the saved workflow. workflow = stored_workflow.latest_workflow @@ -838,7 +838,7 @@ class AdminToolshed( AdminGalaxy ): 'repository/get_repository_information?repository_ids=%s&changeset_revisions=%s' % \ ( repository_ids, changeset_revisions ) ) raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) - repo_information_dict = json.from_json_string( raw_text ) + repo_information_dict = json.from_json_string( raw_text ) for encoded_repo_info_dict in repo_information_dict.get( 'repo_info_dicts', [] ): decoded_repo_info_dict = encoding_util.tool_shed_decode( encoded_repo_info_dict ) if not includes_tools: @@ -1651,7 +1651,7 @@ class AdminToolshed( AdminGalaxy ): if 'data_manager' in metadata_dict: new_data_managers = data_manager_util.install_data_managers( trans.app, trans.app.config.shed_data_manager_config_file, - metadata_dict, + metadata_dict, repository.get_shed_config_dict( trans.app ), os.path.join( relative_install_dir, name ), repository, diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index 45da3ab9844..153819b1e3a 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -32,7 +32,7 @@ class ASync( BaseUIController ): return trans.response.send_redirect( "/index" ) history = trans.get_history( create=True ) - params = Params(kwd, sanitize=False) + params = Params(kwd, sanitize=False) STATUS = params.STATUS URL = params.URL data_id = params.data_id @@ -45,7 +45,7 @@ class ASync( BaseUIController ): tool = toolbox.get_tool( tool_id ) if not tool: return "Tool with id %s not found" % tool_id - + # # we have an incoming data_id # @@ -53,7 +53,7 @@ class ASync( BaseUIController ): if not URL: return "No URL parameter was submitted for data %s" % data_id data = trans.sa_session.query( trans.model.HistoryDatasetAssociation ).get( data_id ) - + if not data: return "Data %s does not exist or has already been deleted" % data_id @@ -76,12 +76,12 @@ class ASync( BaseUIController ): trans.log_event( 'Async error -> %s' % STATUS ) data.state = data.blurb = jobs.JOB_ERROR data.info = "Error -> %s" % STATUS - + trans.sa_session.flush() - + return "Data %s with status %s received. OK" % (data_id, STATUS) else: - # + # # no data_id must be parameter submission # if params.data_type: @@ -90,11 +90,11 @@ class ASync( BaseUIController ): GALAXY_TYPE = 'wig' else: GALAXY_TYPE = params.GALAXY_TYPE or tool.outputs.values()[0].format - + GALAXY_NAME = params.name or params.GALAXY_NAME or '%s query' % tool.name GALAXY_INFO = params.info or params.GALAXY_INFO or params.galaxyDescription or '' GALAXY_BUILD = params.dbkey or params.GALAXY_BUILD or params.galaxyFreeze or '?' - + #data = datatypes.factory(ext=GALAXY_TYPE)() #data.ext = GALAXY_TYPE #data.name = GALAXY_NAME @@ -102,7 +102,7 @@ class ASync( BaseUIController ): #data.dbkey = GALAXY_BUILD #data.state = jobs.JOB_OK #history.datasets.add_dataset( data ) - + data = trans.app.model.HistoryDatasetAssociation( create_dataset=True, sa_session=trans.sa_session, extension=GALAXY_TYPE ) trans.app.security_agent.set_all_dataset_permissions( data.dataset, trans.app.security_agent.history_get_default_permissions( trans.history ) ) data.name = GALAXY_NAME @@ -132,7 +132,7 @@ class ASync( BaseUIController ): except Exception, e: data.info = str(e) data.state = data.blurb = data.states.ERROR - + trans.sa_session.flush() return trans.fill_template( 'tool_executed.mako', history=history, toolbox=toolbox, tool=tool, util=util, out_data={} ) diff --git a/lib/galaxy/webapps/galaxy/controllers/data_admin.py b/lib/galaxy/webapps/galaxy/controllers/data_admin.py index de13afa3284..82762f1d068 100644 --- a/lib/galaxy/webapps/galaxy/controllers/data_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/data_admin.py @@ -19,8 +19,8 @@ except: import logging log = logging.getLogger( __name__ ) -class DataAdmin( BaseUIController ): - jobstyles = dict( +class DataAdmin( BaseUIController ): + jobstyles = dict( done='panel-done-message', waiting='state-color-waiting', running='state-color-running', @@ -30,7 +30,7 @@ class DataAdmin( BaseUIController ): error='panel-error-message', queued='state-color-waiting' ) - + @web.expose @web.require_admin def manage_data( self, trans, **kwd ): @@ -74,7 +74,7 @@ class DataAdmin( BaseUIController ): jobgrid.append( dict( jobtype=jobtype, indexers=indexers, rowclass=state, deferred=job.deferred.id, state=state, intname=job.deferred.params[ 'intname' ], dbkey=job.deferred.params[ 'dbkey' ] ) ) styles = dict( Generate=self.jobstyles['new'], Generated=self.jobstyles['ok'], Disabled=self.jobstyles['error'] ) return trans.fill_template( '/admin/data_admin/local_data.mako', jobgrid=jobgrid, indextable=indextable, labels=labels, dbkeys=dbkeys, styles=styles, indexfuncs=indexfuncs ) - + @web.expose @web.require_admin def add_genome( self, trans, **kwd ): @@ -84,7 +84,7 @@ class DataAdmin( BaseUIController ): ensemblkeys = trans.ensembl_builds ncbikeys = trans.ncbi_builds return trans.fill_template( '/admin/data_admin/data_form.mako', dbkeys=dbkeys, ensembls=ensemblkeys, ncbi=ncbikeys ) - + @web.expose @web.require_admin def genome_search( self, trans, **kwd ): @@ -114,7 +114,7 @@ class DataAdmin( BaseUIController ): intname = params.get( 'longname', None ) indexjob = trans.app.job_manager.deferred_job_queue.plugins['GenomeIndexPlugin'].create_job( trans, path, indexes, dbkey, intname ) return indexjob - + @web.expose @web.require_admin def download_build( self, trans, **kwd ): @@ -149,21 +149,21 @@ class DataAdmin( BaseUIController ): return trans.response.send_redirect( web.url_for( controller='data_admin', action='monitor_status', job=jobid ) ) - + @web.expose @web.require_admin def monitor_status( self, trans, **kwd ): params = util.Params( kwd ) jobid = params.get( 'job', '' ) deferred = trans.app.model.context.current.query( model.DeferredJob ).filter_by( id=jobid ).first() - if deferred is None: + if deferred is None: return trans.fill_template( '/admin/data_admin/generic_error.mako', message='Invalid genome downloader job specified.' ) gname = deferred.params[ 'intname' ] indexers = ', '.join( deferred.params[ 'indexes' ] ) jobs = self._get_jobs( deferred, trans ) jsonjobs = simplejson.dumps( jobs ) return trans.fill_template( '/admin/data_admin/download_status.mako', name=gname, indexers=indexers, mainjob=jobid, jobs=jobs, jsonjobs=jsonjobs ) - + @web.expose @web.require_admin def get_jobs( self, trans, **kwd ): @@ -174,7 +174,7 @@ class DataAdmin( BaseUIController ): job = sa_session.query( model.DeferredJob ).filter_by( id=jobid ).first() jobs = self._get_jobs( job, trans ) return trans.fill_template( '/admin/data_admin/ajax_status.mako', json=simplejson.dumps( jobs ) ) - + def _get_job( self, jobid, jobtype, trans ): sa = trans.app.model.context.current if jobtype == 'liftover': @@ -191,7 +191,7 @@ class DataAdmin( BaseUIController ): job = sa.query( model.Job ).filter_by( id=jobid.job_id ).first() joblabel = 'Index Genome (%s)' % jobid.indexer return dict( status=job.state, jobid=job.id, style=self.jobstyles[job.state], type=jobtype, label=joblabel ) - + def _get_jobs( self, deferredjob, trans ): jobs = [] idxjobs = [] @@ -210,7 +210,7 @@ class DataAdmin( BaseUIController ): return jobs def build_param_dict( params, trans ): - + source = params.get('source', '') longname = params.get('longname', None) if not isinstance( params.get( 'indexers', None ), list ): @@ -226,7 +226,7 @@ def build_param_dict( params, trans ): dbkey = params.get( 'dbkey', None ) dbkeys = dict() protocol = 'http' - + if source == 'NCBI': build = params.get('ncbi_name', '') dbkey = build.split( ': ' )[0] @@ -242,7 +242,7 @@ def build_param_dict( params, trans ): if dbkey == build[0]: dbkey = build[0] longname = build[1] - break + break if dbkey == '?': return dict( status='error', message='An invalid build was specified.' ) ftp = ftplib.FTP('hgdownload.cse.ucsc.edu') @@ -281,7 +281,7 @@ def build_param_dict( params, trans ): message = 'The genome %s was not found on the UCSC server.' % dbkey status = 'error' return dict( status=status, message=message ) - + elif source == 'Ensembl': dbkey = params.get( 'ensembl_dbkey', None ) if dbkey == '?': @@ -294,7 +294,7 @@ def build_param_dict( params, trans ): longname = build[ 'name' ].replace('_', ' ') break url = 'ftp://ftp.ensembl.org/pub/release-%s/fasta/%s/dna/%s.%s.%s.dna.toplevel.fa.gz' % ( release, pathname.lower(), pathname, dbkey, release ) - + params = dict( status='ok', dbkey=dbkey, datatype='fasta', url=url, user=trans.user.id, liftover=newlift, longname=longname, indexers=indexers ) return params \ No newline at end of file diff --git a/lib/galaxy/webapps/galaxy/controllers/data_manager.py b/lib/galaxy/webapps/galaxy/controllers/data_manager.py index 18d4ad9442d..0837234c1bb 100644 --- a/lib/galaxy/webapps/galaxy/controllers/data_manager.py +++ b/lib/galaxy/webapps/galaxy/controllers/data_manager.py @@ -10,8 +10,8 @@ import paste.httpexceptions import logging log = logging.getLogger( __name__ ) -class DataManager( BaseUIController ): - +class DataManager( BaseUIController ): + @web.expose def index( self, trans, **kwd ): not_is_admin = not trans.user_is_admin() @@ -20,7 +20,7 @@ class DataManager( BaseUIController ): message = kwd.get( 'message' ) status = kwd.get( 'status', 'info' ) return trans.fill_template( "data_manager/index.mako", data_managers=trans.app.data_managers, view_only=not_is_admin, message=message, status=status ) - + @web.expose def manage_data_manager( self, trans, **kwd ): not_is_admin = not trans.user_is_admin() @@ -34,7 +34,7 @@ class DataManager( BaseUIController ): return trans.response.send_redirect( web.url_for( controller="data_manager", action="index", message="Invalid Data Manager (%s) was requested" % data_manager_id, status="error" ) ) jobs = reversed( [ assoc.job for assoc in trans.sa_session.query( trans.app.model.DataManagerJobAssociation ).filter_by( data_manager_id=data_manager_id ) ] ) return trans.fill_template( "data_manager/manage_data_manager.mako", data_manager=data_manager, jobs=jobs, view_only=not_is_admin, message=message, status=status ) - + @web.expose def view_job( self, trans, **kwd ): not_is_admin = not trans.user_is_admin() @@ -60,7 +60,7 @@ class DataManager( BaseUIController ): values = [] for key, value in data_manager_json.get( 'data_tables', {} ).iteritems(): values.append( ( key, value ) ) - data_manager_output.append( values ) + data_manager_output.append( values ) return trans.fill_template( "data_manager/view_job.mako", data_manager=data_manager, job=job, view_only=not_is_admin, hdas=hdas, data_manager_output=data_manager_output, message=message, status=status ) @web.expose diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index b22b86dfdcb..28405e0acda 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -157,13 +157,13 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, Use hda = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( dataset_id ) ) assert hda and self._can_access_dataset( trans, hda ) return hda.creating_job - + def _can_access_dataset( self, trans, dataset_association, allow_admin=True, additional_roles=None ): roles = trans.get_current_user_roles() if additional_roles: roles = roles + additional_roles return ( allow_admin and trans.user_is_admin() ) or trans.app.security_agent.can_access_dataset( roles, dataset_association.dataset ) - + @web.expose def errors( self, trans, id ): try: @@ -185,7 +185,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, Use stdout = job.stdout except: stdout = "Invalid dataset ID or you are not allowed to access this dataset" - return stdout + return stdout @web.expose # TODO: Migrate stderr and stdout to use _get_job_for_dataset; it wasn't tested. @@ -197,7 +197,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, Use stderr = job.stderr except: stderr = "Invalid dataset ID or you are not allowed to access this dataset" - return stderr + return stderr @web.expose def exit_code( self, trans, dataset_id=None, **kwargs ): @@ -208,7 +208,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, Use exit_code = job.exit_code except: exit_code = "Invalid dataset ID or you are not allowed to access this dataset" - return exit_code + return exit_code @web.expose def report_error( self, trans, id, email='', message="", **kwd ): smtp_server = trans.app.config.smtp_server @@ -549,7 +549,7 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, Use # target_histories = [ trans.get_history() ] - + # Reverse HDAs so that they appear in the history in the order they are provided. hda_ids.reverse() status, message = self._copy_datasets( trans, hda_ids, target_histories ) @@ -652,8 +652,8 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, Use user_item_rating = 0 ave_item_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, dataset ) - return trans.fill_template_mako( "/dataset/display.mako", item=dataset, item_data=dataset_data, - truncated=truncated, user_item_rating = user_item_rating, + return trans.fill_template_mako( "/dataset/display.mako", item=dataset, item_data=dataset_data, + truncated=truncated, user_item_rating = user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings, first_chunk=first_chunk ) else: diff --git a/lib/galaxy/webapps/galaxy/controllers/external_service.py b/lib/galaxy/webapps/galaxy/controllers/external_service.py index b7bc111d3cd..83aaf8836d1 100644 --- a/lib/galaxy/webapps/galaxy/controllers/external_service.py +++ b/lib/galaxy/webapps/galaxy/controllers/external_service.py @@ -32,7 +32,7 @@ class ExternalServiceGrid( grids.Grid ): use_paging = True default_filter = dict( deleted="False" ) columns = [ - NameColumn( "Name", + NameColumn( "Name", key="name", link=( lambda item: iff( item.deleted, None, dict( operation="view", id=item.id ) ) ), attach_popup=True, @@ -41,23 +41,23 @@ class ExternalServiceGrid( grids.Grid ): key='description', filterable="advanced" ), ExternalServiceTypeColumn( "External Service Type" ), - grids.GridColumn( "Last Updated", - key="update_time", + grids.GridColumn( "Last Updated", + key="update_time", format=time_ago ), - grids.DeletedColumn( "Deleted", - key="deleted", - visible=False, + grids.DeletedColumn( "Deleted", + key="deleted", + visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) operations = [ grids.GridOperation( "Edit", allow_multiple=False, condition=( lambda item: not item.deleted ) ), grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted ) ), - grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), + grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), ] global_actions = [ grids.GridAction( "Reload external service types", dict( controller='external_service', action='reload_external_service_types' ) ), @@ -131,7 +131,7 @@ class ExternalService( BaseUIController, UsesFormDefinitionsMixin ): except: return invalid_id_redirect( trans, 'external_service', external_service_id, 'external_service', action='browse_external_services' ) external_service_type = self.get_external_service_type( trans, external_service.external_service_type_id ) - return trans.fill_template( '/admin/external_service/view_external_service.mako', + return trans.fill_template( '/admin/external_service/view_external_service.mako', external_service=external_service, external_service_type=external_service_type ) @web.expose @@ -271,9 +271,9 @@ class ExternalService( BaseUIController, UsesFormDefinitionsMixin ): if params.get( 'reload_external_service_type_button', False ): new_external_service_type = trans.app.external_service_types.reload( external_service_type_id ) status = 'done' - message = 'Reloaded external service type: %s' % new_external_service_type.name - external_service_type_select_field = self.__build_external_service_type_select_field( trans, - external_service_type_id, + message = 'Reloaded external service type: %s' % new_external_service_type.name + external_service_type_select_field = self.__build_external_service_type_select_field( trans, + external_service_type_id, refresh_on_change=False, visible_external_service_types_only=False ) if not trans.app.external_service_types.visible_external_service_types: @@ -283,7 +283,7 @@ class ExternalService( BaseUIController, UsesFormDefinitionsMixin ): action='browse_external_services', message=message, status=status ) ) - return trans.fill_template( '/admin/external_service/reload_external_service_types.mako', + return trans.fill_template( '/admin/external_service/reload_external_service_types.mako', external_service_type_select_field=external_service_type_select_field, message=message, status=status ) @@ -315,13 +315,13 @@ class ExternalService( BaseUIController, UsesFormDefinitionsMixin ): else: seq_type = 'none' widgets = [ dict( label='Name', - widget=TextField( 'external_service_name', 40, name ), + widget=TextField( 'external_service_name', 40, name ), helptext='' ), dict( label='Description', - widget=TextField( 'external_service_description', 40, description ), + widget=TextField( 'external_service_description', 40, description ), helptext='' ), dict( label='Version', - widget=TextField( 'external_service_version', 40, version ), + widget=TextField( 'external_service_version', 40, version ), helptext='' ) ] # Do not show the external_service_type selectfield when editing a external_service if not external_service: @@ -335,10 +335,10 @@ class ExternalService( BaseUIController, UsesFormDefinitionsMixin ): objs_list = [ external_service_types[ seq_type_id ] for seq_type_id in trans.app.external_service_types.visible_external_service_types ] else: objs_list = external_service_types.values() - refresh_on_change_values = [ 'none' ] + refresh_on_change_values = [ 'none' ] refresh_on_change_values.extend( [ trans.security.encode_id( obj.id ) for obj in objs_list] ) - select_external_service_type = SelectField( 'external_service_type_id', - refresh_on_change=refresh_on_change, + select_external_service_type = SelectField( 'external_service_type_id', + refresh_on_change=refresh_on_change, refresh_on_change_values=refresh_on_change_values ) if selected_value == 'none': select_external_service_type.add_option( 'Select one', 'none', selected=True ) diff --git a/lib/galaxy/webapps/galaxy/controllers/forms.py b/lib/galaxy/webapps/galaxy/controllers/forms.py index ae7b5b094ce..ce8f8907f4c 100644 --- a/lib/galaxy/webapps/galaxy/controllers/forms.py +++ b/lib/galaxy/webapps/galaxy/controllers/forms.py @@ -3,8 +3,8 @@ from galaxy.model.orm import * from galaxy.datatypes import sniff from galaxy import model, util import logging, os, sys -from galaxy.web.form_builder import * -from galaxy.tools.parameters.basic import parameter_types +from galaxy.web.form_builder import * +from galaxy.tools.parameters.basic import parameter_types from elementtree.ElementTree import XML, Element from galaxy.util.odict import odict import copy @@ -35,32 +35,32 @@ class FormsGrid( grids.Grid ): use_paging = True default_filter = dict( deleted="False" ) columns = [ - NameColumn( "Name", - key="name", + NameColumn( "Name", + key="name", model_class=model.FormDefinition, - link=( lambda item: iff( item.deleted, None, dict( operation="view_latest_form_definition", + link=( lambda item: iff( item.deleted, None, dict( operation="view_latest_form_definition", id=item.id ) ) ), - attach_popup=True, + attach_popup=True, filterable="advanced" ), DescriptionColumn( "Description", key='desc', model_class=model.FormDefinition, filterable="advanced" ), TypeColumn( "Type" ), - grids.DeletedColumn( "Deleted", - key="deleted", - visible=False, + grids.DeletedColumn( "Deleted", + key="deleted", + visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) operations = [ grids.GridOperation( "Edit", allow_multiple=False, condition=( lambda item: not item.deleted ) ), grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted ) ), - grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), + grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), ] global_actions = [ grids.GridAction( "Create new form", dict( controller='forms', action='create_form_definition' ) ) @@ -68,9 +68,9 @@ class FormsGrid( grids.Grid ): class Forms( BaseUIController ): # Empty TextField - empty_field = { 'name': '', - 'label': '', - 'helptext': '', + empty_field = { 'name': '', + 'label': '', + 'helptext': '', 'visible': True, 'required': False, 'type': model.TextField.__name__, @@ -120,7 +120,7 @@ class Forms( BaseUIController ): message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) self.__imported_from_file = False - if params.get( 'create_form_button', False ): + if params.get( 'create_form_button', False ): form_definition, message = self.save_form_definition( trans, form_definition_current_id=None, **kwd ) if not form_definition: return trans.response.send_redirect( web.url_for( controller='forms', @@ -132,7 +132,7 @@ class Forms( BaseUIController ): if self.__imported_from_file: return trans.response.send_redirect( web.url_for( controller='forms', action='edit_form_definition', - id=trans.security.encode_id( form_definition.current.id )) ) + id=trans.security.encode_id( form_definition.current.id )) ) else: return trans.response.send_redirect( web.url_for( controller='forms', action='edit_form_definition', @@ -140,12 +140,12 @@ class Forms( BaseUIController ): add_field_button='Add field', name=form_definition.name, description=form_definition.desc, - form_type_select_field=form_definition.type ) ) + form_type_select_field=form_definition.type ) ) inputs = [ ( 'Name', TextField( 'name', 40, util.restore_text( params.get( 'name', '' ) ) ) ), ( 'Description', TextField( 'description', 40, util.restore_text( params.get( 'description', '' ) ) ) ), ( 'Type', self.__build_form_types_widget( trans, selected=params.get( 'form_type', 'none' ) ) ), ( 'Import from csv file (Optional)', FileField( 'file_data', 40, '' ) ) ] - return trans.fill_template( '/admin/forms/create_form.mako', + return trans.fill_template( '/admin/forms/create_form.mako', inputs=inputs, message=message, status=status ) @@ -155,7 +155,7 @@ class Forms( BaseUIController ): ''' This callback method is for handling form editing. The value of response_redirect should be an URL that is defined by the caller. This allows for redirecting as desired - when the form changes have been saved. For an example of how this works, see the + when the form changes have been saved. For an example of how this works, see the edit_template() method in the base controller. ''' params = util.Params( kwd ) @@ -171,7 +171,7 @@ class Forms( BaseUIController ): form_definition = form_definition_current.latest_form # TODO: eliminate the need for this refresh param. if params.get( 'refresh', False ): - # Refresh + # Refresh current_form = self.get_current_form( trans, **kwd ) else: # Show the saved form for editing @@ -179,7 +179,7 @@ class Forms( BaseUIController ): # Save changes if params.get( 'save_changes_button', False ): new_form_definition, message = self.save_form_definition( trans, form_definition_current_id=form_definition.form_definition_current.id, **kwd ) - # if validation error encountered while saving the form, show the + # if validation error encountered while saving the form, show the # unsaved form, with the error message if not new_form_definition: status = 'error' @@ -211,45 +211,45 @@ class Forms( BaseUIController ): del current_form[ 'fields' ][ index ] # Add SelectField option elif 'Add' in kwd.values(): - current_form, status, message = self.__add_select_field_option( trans=trans, + current_form, status, message = self.__add_select_field_option( trans=trans, current_form=current_form, **kwd) # Remove SelectField option elif 'Remove' in kwd.values(): - current_form, status, message = self.__remove_select_field_option( trans=trans, + current_form, status, message = self.__remove_select_field_option( trans=trans, current_form=current_form, **kwd) return self.show_editable_form_definition( trans=trans, form_definition=form_definition, - current_form=current_form, + current_form=current_form, message=message, status=status, response_redirect=response_redirect, **kwd ) def get_saved_form( self, form_definition ): ''' - This retrieves the saved form and returns a dictionary containing the name, + This retrieves the saved form and returns a dictionary containing the name, desc, type, layout & fields of the form ''' if form_definition.type == form_definition.types.SAMPLE: return dict( name=form_definition.name, desc=form_definition.desc, type=form_definition.type, - layout=list( copy.deepcopy( form_definition.layout ) ), + layout=list( copy.deepcopy( form_definition.layout ) ), fields=list( copy.deepcopy( form_definition.fields ) ) ) return dict( name=form_definition.name, desc=form_definition.desc, type=form_definition.type, - layout=[], + layout=[], fields=list( copy.deepcopy( form_definition.fields ) ) ) def get_current_form( self, trans, **kwd ): ''' - This method gets all the unsaved user-entered form details and returns a + This method gets all the unsaved user-entered form details and returns a dictionary containing the name, desc, type, layout & fields of the form ''' params = util.Params( kwd ) - name = util.restore_text( params.name ) + name = util.restore_text( params.name ) desc = util.restore_text( params.description ) or "" form_type = util.restore_text( params.form_type_select_field ) # get the user entered layout grids in it is a sample form definition @@ -285,7 +285,7 @@ class Forms( BaseUIController ): fields = fields) def save_form_definition( self, trans, form_definition_current_id=None, **kwd ): ''' - This method saves the current form + This method saves the current form ''' # check the form for invalid inputs flag, message = self.__validate_form( **kwd ) @@ -293,7 +293,7 @@ class Forms( BaseUIController ): return None, message current_form = self.get_current_form( trans, **kwd ) # validate fields - field_names_dict = {} + field_names_dict = {} for field in current_form[ 'fields' ]: if not field[ 'label' ]: return None, "All the field labels must be completed." @@ -307,14 +307,14 @@ class Forms( BaseUIController ): if current_form[ 'type' ] == trans.app.model.FormDefinition.types.SAMPLE and not len( current_form[ 'layout' ] ): current_form[ 'layout' ] = [ 'Layout1' ] # create a new form definition - form_definition = trans.app.model.FormDefinition( name=current_form[ 'name' ], - desc=current_form[ 'desc' ], - fields=current_form[ 'fields' ], - form_definition_current=None, - form_type=current_form[ 'type' ], + form_definition = trans.app.model.FormDefinition( name=current_form[ 'name' ], + desc=current_form[ 'desc' ], + fields=current_form[ 'fields' ], + form_definition_current=None, + form_type=current_form[ 'type' ], layout=current_form[ 'layout' ] ) - if form_definition_current_id: # save changes to the existing form - # change the pointer in the form_definition_current table to point + if form_definition_current_id: # save changes to the existing form + # change the pointer in the form_definition_current table to point # to this new record form_definition_current = trans.sa_session.query( trans.app.model.FormDefinitionCurrent ).get( form_definition_current_id ) else: # create a new form @@ -328,8 +328,8 @@ class Forms( BaseUIController ): return form_definition, message def show_editable_form_definition( self, trans, form_definition, current_form, message='', status='done', response_redirect=None, **kwd ): """ - Displays the form and any of the changes made to it in edit mode. In this method - all the widgets are build for all name, description and all the fields of a form + Displays the form and any of the changes made to it in edit mode. In this method + all the widgets are build for all name, description and all the fields of a form definition. """ params = util.Params( kwd ) @@ -340,14 +340,14 @@ class Forms( BaseUIController ): form_layout = [] if current_form[ 'type' ] == trans.app.model.FormDefinition.types.SAMPLE: for index, layout_name in enumerate( current_form[ 'layout' ] ): - form_layout.append( TextField( 'grid_layout%i' % index, 40, layout_name )) + form_layout.append( TextField( 'grid_layout%i' % index, 40, layout_name )) # fields field_details = [] for field_index, field in enumerate( current_form[ 'fields' ] ): - field_widgets = self.build_form_definition_field_widgets( trans=trans, - layout_grids=current_form['layout'], - field_index=field_index, - field=field, + field_widgets = self.build_form_definition_field_widgets( trans=trans, + layout_grids=current_form['layout'], + field_index=field_index, + field=field, form_type=current_form['type'] ) field_details.append( field_widgets ) return trans.fill_template( '/admin/forms/edit_form_definition.mako', @@ -361,7 +361,7 @@ class Forms( BaseUIController ): layout_grids=form_layout, response_redirect=response_redirect ) @web.expose - @web.require_admin + @web.require_admin def delete_form_definition( self, trans, **kwd ): id_list = util.listify( kwd['id'] ) delete_failed = [] @@ -378,7 +378,7 @@ class Forms( BaseUIController ): trans.sa_session.flush() return trans.response.send_redirect( web.url_for( controller='forms', action='browse_form_definitions', - message='%i forms have been deleted.' % len(id_list), + message='%i forms have been deleted.' % len(id_list), status='done') ) @web.expose @web.require_admin @@ -398,11 +398,11 @@ class Forms( BaseUIController ): trans.sa_session.flush() return trans.response.send_redirect( web.url_for( controller='forms', action='browse_form_definitions', - message='%i forms have been undeleted.' % len(id_list), + message='%i forms have been undeleted.' % len(id_list), status='done') ) def build_form_definition_field_widgets( self, trans, layout_grids, field_index, field, form_type ): ''' - This method returns a list of widgets which describes a form definition field. This + This method returns a list of widgets which describes a form definition field. This includes the field label, helptext, type, selectfield options, required/optional & layout ''' # field label @@ -410,8 +410,8 @@ class Forms( BaseUIController ): # help text helptext = TextField( 'field_helptext_'+str( field_index ), 40, field['helptext'] ) # field type - field_type_select_field = SelectField( 'field_type_'+str( field_index ), - refresh_on_change=True, + field_type_select_field = SelectField( 'field_type_'+str( field_index ), + refresh_on_change=True, refresh_on_change_values=[ SelectField.__name__ ] ) # fill up the field type selectfield options field_type_options = [] @@ -420,26 +420,26 @@ class Forms( BaseUIController ): if form_type == trans.model.FormDefinition.types.SAMPLE: for supported_field_type in trans.model.Sample.supported_field_types: if supported_field_type.__name__ == field[ 'type' ]: - field_type_select_field.add_option( supported_field_type.__name__, - supported_field_type.__name__, + field_type_select_field.add_option( supported_field_type.__name__, + supported_field_type.__name__, selected=True ) if supported_field_type.__name__ == SelectField.__name__: - # when field type is Selectfield, add option Textfields + # when field type is Selectfield, add option Textfields field_type_options = self.__build_field_type_select_field_options( field, field_index ) else: - field_type_select_field.add_option( supported_field_type.__name__, + field_type_select_field.add_option( supported_field_type.__name__, supported_field_type.__name__ ) else: for supported_field_type in trans.model.FormDefinition.supported_field_types: if supported_field_type.__name__ == field[ 'type' ]: - field_type_select_field.add_option( supported_field_type.__name__, - supported_field_type.__name__, + field_type_select_field.add_option( supported_field_type.__name__, + supported_field_type.__name__, selected=True ) if supported_field_type.__name__ == SelectField.__name__: - # when field type is Selectfield, add option Textfields + # when field type is Selectfield, add option Textfields field_type_options = self.__build_field_type_select_field_options( field, field_index ) else: - field_type_select_field.add_option( supported_field_type.__name__, + field_type_select_field.add_option( supported_field_type.__name__, supported_field_type.__name__ ) # required/optional radio button required = SelectField( 'field_required_'+str(field_index), display='radio' ) @@ -459,8 +459,8 @@ class Forms( BaseUIController ): grid_selected = False layout_select_field.add_option("%i. %s" %( index+1, grid_name ), index, selected=grid_selected ) # default value - default_value = TextField( 'field_default_'+str(field_index), - 40, + default_value = TextField( 'field_default_'+str(field_index), + 40, field.get( 'default', '' ) ) # field name name = TextField( 'field_name_' + str( field_index ), 40, field[ 'name' ] ) @@ -494,7 +494,7 @@ class Forms( BaseUIController ): This method adds a select_field option. The kwd dict searched for the field index which needs to be removed ''' - message='' + message='' status='ok', index = -1 for k, v in kwd.items(): @@ -505,7 +505,7 @@ class Forms( BaseUIController ): break if index == -1: # something wrong happened - message='Error in adding selectfield option', + message='Error in adding selectfield option', status='error', return current_form, status, message # add an empty option @@ -516,7 +516,7 @@ class Forms( BaseUIController ): This method removes a select_field option. The kwd dict searched for the field index and option index which needs to be removed ''' - message='' + message='' status='ok', option = -1 for k, v in kwd.items(): @@ -528,7 +528,7 @@ class Forms( BaseUIController ): break if option == -1: # something wrong happened - message='Error in removing selectfield option', + message='Error in removing selectfield option', status='error', return current_form, status, message # remove the option @@ -537,14 +537,14 @@ class Forms( BaseUIController ): def __get_select_field_options( self, index, **kwd ): ''' This method gets all the options entered by the user for field when - the fieldtype is SelectField + the fieldtype is SelectField ''' params = util.Params( kwd ) ctr=0 sb_options = [] while True: if kwd.has_key( 'field_'+str(index)+'_option_'+str(ctr) ): - option = params.get( 'field_'+str(index)+'_option_'+str(ctr), None ) + option = params.get( 'field_'+str(index)+'_option_'+str(ctr), None ) sb_options.append( util.restore_text( option ) ) ctr = ctr+1 else: @@ -567,8 +567,8 @@ class Forms( BaseUIController ): if field_type == 'SelectField': options = self.__get_select_field_options(index, **kwd) return { 'name': name, - 'label': label, - 'helptext': helptext, + 'label': label, + 'helptext': helptext, 'visible': True, 'required': required, 'type': field_type, @@ -576,8 +576,8 @@ class Forms( BaseUIController ): 'layout': layout, 'default': default } return { 'name': name, - 'label': label, - 'helptext': helptext, + 'label': label, + 'helptext': helptext, 'visible': True, 'required': required, 'type': field_type, @@ -600,19 +600,19 @@ class Forms( BaseUIController ): options = row[5].split(',') if len(row) >= 8: fields.append( { 'name': '%i_field_name' % index, - 'label': row[0], - 'helptext': row[1], + 'label': row[0], + 'helptext': row[1], 'visible': row[2], 'required': row[3], 'type': row[4], 'selectlist': options, 'layout':row[6], 'default': row[7] } ) - layouts.add(row[6]) + layouts.add(row[6]) else: fields.append( { 'name': '%i_field_name' % index, - 'label': row[0], - 'helptext': row[1], + 'label': row[0], + 'helptext': row[1], 'visible': row[2], 'required': row[3], 'type': row[4], @@ -637,7 +637,7 @@ class Forms( BaseUIController ): if not util.restore_text( params.name ): return None, 'Form name must be filled.' # form type - if util.restore_text( params.form_type_select_field ) == 'none': + if util.restore_text( params.form_type_select_field ) == 'none': return None, 'Form type must be selected.' return True, '' def __build_form_types_widget( self, trans, selected='none' ): diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index e2bfe4c5efd..72f9cce24fd 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -526,7 +526,7 @@ class HistoryController( BaseUIController, SharableMixin, UsesAnnotations, UsesI if hda.dataset.user_can_purge: try: hda.dataset.full_delete() - trans.log_event( "Dataset id %s has been purged upon the the purge of HDA id %s" % ( hda.dataset.id, hda.id ) ) + trans.log_event( "Dataset id %s has been purged upon the the purge of HDA id %s" % ( hda.dataset.id, hda.id ) ) trans.sa_session.add( hda.dataset ) except: log.exception( 'Unable to purge dataset (%s) on purge of hda (%s):' % ( hda.dataset.id, hda.id ) ) @@ -1312,7 +1312,7 @@ class HistoryController( BaseUIController, SharableMixin, UsesAnnotations, UsesI copy_choice = params.get( 'copy_choice', None ) if not copy_choice: return trans.fill_template( "/history/copy.mako", id_argument=id ) - + # Extract histories for id argument, defaulting to current if id is None: histories = [ trans.history ] @@ -1352,7 +1352,7 @@ class HistoryController( BaseUIController, SharableMixin, UsesAnnotations, UsesI history = self.get_history( trans, hist_id ) trans.set_history( history ) return trans.response.send_redirect( url_for( "/" ) ) - + def get_item( self, trans, id ): return self.get_history( trans, id ) diff --git a/lib/galaxy/webapps/galaxy/controllers/library_admin.py b/lib/galaxy/webapps/galaxy/controllers/library_admin.py index eb831d22e4d..7640fbc0be6 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_admin.py @@ -49,7 +49,7 @@ class LibraryListGrid( grids.Grid ): grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ) ] columns.append( grids.MulticolFilterColumn( "search dataset name, info, message, dbkey", - cols_to_filter=[ columns[0], columns[1] ], + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -237,4 +237,4 @@ class LibraryAdmin( BaseUIController ): return trans.response.send_redirect( web.url_for( controller='library_admin', action='browse_libraries', message=galaxy.util.sanitize_text( message ), - status='done' ) ) + status='done' ) ) diff --git a/lib/galaxy/webapps/galaxy/controllers/library_common.py b/lib/galaxy/webapps/galaxy/controllers/library_common.py index 32f30d53046..d15ddbe7bc9 100644 --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -566,8 +566,8 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet self.delete_extended_metadata(trans, ex_obj) message = "Deleted Extended metadata '%s'." % ldda.name status = 'done' - - + + if "dbkey" in ldda.datatype.metadata_spec and not ldda.metadata.dbkey: # Copy dbkey into metadata, for backwards compatability @@ -614,7 +614,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet .all() else: associated_hdas = [] - associated_lddas = [] + associated_lddas = [] # See if we have any associated templates widgets = [] widget_fields_have_contents = False @@ -949,7 +949,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet default_action=default_action, created_ldda_ids=created_ldda_ids, show_deleted=show_deleted, - message=util.sanitize_text( message ), + message=util.sanitize_text( message ), status='done' ) ) else: created_ldda_ids = '' @@ -968,8 +968,8 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet status=status ) ) # Note: if the upload form was submitted due to refresh_on_change for a form field, we cannot re-populate # the field for the selected file ( files_0|file_data ) if the user selected one. This is because the value - # attribute of the html input file type field is typically ignored by browsers as a security precaution. - + # attribute of the html input file type field is typically ignored by browsers as a security precaution. + # See if we have any inherited templates. if not info_association: info_association, inherited = folder.get_info_association( inherited=True ) @@ -979,7 +979,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet # Retain contents of widget fields when form was submitted via refresh_on_change. widgets = self.populate_widgets_from_kwd( trans, widgets, **kwd ) template_id = str( info_association.template.id ) - + # Send list of data formats to the upload form so the "extension" select list can be populated dynamically file_formats = trans.app.datatypes_registry.upload_file_formats @@ -1332,7 +1332,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet created_ldda_ids = '%s,%s' % ( created_ldda_ids, str( ldda.id ) ) dataset_names.append( ldda.name ) if not replace_dataset: - # If replace_dataset is None, the Library level permissions will be taken from the folder and applied to the new + # If replace_dataset is None, the Library level permissions will be taken from the folder and applied to the new # LDDA and LibraryDataset. trans.app.security_agent.copy_library_permissions( trans, folder, ldda ) trans.app.security_agent.copy_library_permissions( trans, folder, ldda.library_dataset ) @@ -1469,8 +1469,8 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet for option_value, option_label in trans.model.LibraryDataset.upload_options: if option_value not in do_not_include_values: upload_refresh_on_change_values.append( option_value ) - upload_option_select_list = SelectField( 'upload_option', - refresh_on_change=True, + upload_option_select_list = SelectField( 'upload_option', + refresh_on_change=True, refresh_on_change_values=upload_refresh_on_change_values ) for option_value, option_label in trans.model.LibraryDataset.upload_options: if option_value not in do_not_include_values: @@ -1504,7 +1504,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet def download_dataset_from_folder( self, trans, cntrller, id, library_id=None, **kwd ): """Catches the dataset id and displays file contents as directed""" show_deleted = util.string_as_bool( kwd.get( 'show_deleted', False ) ) - params = util.Params( kwd ) + params = util.Params( kwd ) use_panels = util.string_as_bool( params.get( 'use_panels', False ) ) is_admin = trans.user_is_admin() and cntrller == 'library_admin' current_user_roles = trans.get_current_user_roles() @@ -1530,7 +1530,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s"' % fname try: return open( ldda.file_name ) - except: + except: message = 'This dataset contains no content' return trans.response.send_redirect( web.url_for( controller='library_common', action='browse_library', @@ -1888,7 +1888,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet if is_composite: # need to add all the components from the extra_files_path to the zip if zpathext == '': - zpath = '%s.html' % zpath # fake the real nature of the html file + zpath = '%s.html' % zpath # fake the real nature of the html file try: archive.add(ldda.dataset.file_name,zpath) # add the primary of a composite set except IOError: @@ -1896,7 +1896,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet log.exception( "Unable to add composite parent %s to temporary library download archive" % ldda.dataset.file_name) message = "Unable to create archive for download, please report this error" status = 'error' - continue + continue flist = glob.glob(os.path.join(ldda.dataset.extra_files_path,'*.*')) # glob returns full paths for fpath in flist: efp,fname = os.path.split(fpath) @@ -1917,7 +1917,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet error = True log.exception( "Unable to write %s to temporary library download archive" % ldda.dataset.file_name) message = "Unable to create archive for download, please report this error" - status = 'error' + status = 'error' if not error: if library_id: lname = trans.sa_session.query( trans.app.model.Library ).get( trans.security.decode_id( library_id ) ).name @@ -1972,13 +1972,13 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet use_panels=use_panels, message=message, status=status ) - + @web.expose def import_datasets_to_histories( self, trans, cntrller, library_id='', folder_id='', ldda_ids='', target_history_id='', target_history_ids='', new_history_name='', **kwd ): # This method is called from one of the following places: # - a menu option for a library dataset ( ldda_ids is a single ldda id ) # - a menu option for a library folder ( folder_id has a value ) - # - a select list option for acting on multiple selected datasets within a library + # - a select list option for acting on multiple selected datasets within a library # ( ldda_ids is a comma separated string of ldda ids ) # - a menu option for a library dataset search result set ( ldda_ids is a comma separated string of ldda ids ) params = util.Params( kwd ) @@ -2150,9 +2150,9 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet # This method is called from one of the following places: # - a menu option for a library dataset ( item_type is 'ldda' and item_id is a single ldda id ) # - a menu option for a library folder ( item_type is 'folder' and item_id is a single folder id ) - # - a select list option for acting on multiple selected datasets within a library ( item_type is + # - a select list option for acting on multiple selected datasets within a library ( item_type is # 'ldda' and item_id is a comma separated string of ldda ids ) - # - a menu option for a library dataset search result set ( item_type is 'ldda' and item_id is a + # - a menu option for a library dataset search result set ( item_type is 'ldda' and item_id is a # comma separated string of ldda ids ) params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) @@ -2478,10 +2478,10 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet status = 'error' if not_authorized_items: message += 'You are not authorized to undelete %d %s. ' % ( not_authorized_items, inflector.cond_plural( not_authorized_items, item_desc ) ) - status = 'error' + status = 'error' if purged_items: message += '%d %s marked purged, so cannot be undeleted. ' % ( purged_items, inflector.cond_plural( purged_items, item_desc ) ) - status = 'error' + status = 'error' if item_type == 'library': return trans.response.send_redirect( web.url_for( controller=cntrller, action='browse_libraries', @@ -2556,7 +2556,7 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet if isinstance( item, trans.model.LibraryDataset ): # Deny access if the user is not an admin and does not have the LIBRARY_MANAGE and DATASET_MANAGE_PERMISSIONS permissions. if not ( is_admin or \ - ( trans.app.security_agent.can_manage_library_item( current_user_roles, item ) and + ( trans.app.security_agent.can_manage_library_item( current_user_roles, item ) and trans.app.security_agent.can_manage_dataset( current_user_roles, library_dataset.library_dataset_dataset_association.dataset ) ) ): message = "You are not authorized to manage permissions on library dataset (%s)." % library_dataset.name if cntrller == 'api': @@ -2619,13 +2619,13 @@ def map_library_datasets_to_lddas( trans, lib_datasets ): there will be no entry in the return hash. ''' # Get a list of the LibraryDatasets' ids so that we can pass it along to - # a query to retrieve the LDDAs. This eliminates querying for each + # a query to retrieve the LDDAs. This eliminates querying for each # LibraryDataset. lib_dataset_ids = [ x.library_dataset_dataset_association_id for x in lib_datasets ] lddas = trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ) \ .filter( trans.app.model.LibraryDatasetDatasetAssociation.id.in_( lib_dataset_ids ) ) \ .all() - + # Map the LibraryDataset to the returned LDDAs: ret_lddas = {} for ldda in lddas: @@ -2634,13 +2634,13 @@ def map_library_datasets_to_lddas( trans, lib_datasets ): def datasets_for_lddas( trans, lddas ): ''' - Given a list of LDDAs, return a list of Datasets for them. + Given a list of LDDAs, return a list of Datasets for them. ''' dataset_ids = [ x.dataset_id for x in lddas ] datasets = trans.sa_session.query( trans.app.model.Dataset ) \ .filter( trans.app.model.Dataset.id.in_( dataset_ids ) ) \ .all() - return datasets + return datasets def active_folders_and_library_datasets( trans, folder ): folders = active_folders( trans, folder ) diff --git a/lib/galaxy/webapps/galaxy/controllers/mobile.py b/lib/galaxy/webapps/galaxy/controllers/mobile.py index e8a72194176..c813d4e3036 100644 --- a/lib/galaxy/webapps/galaxy/controllers/mobile.py +++ b/lib/galaxy/webapps/galaxy/controllers/mobile.py @@ -4,11 +4,11 @@ class Mobile( BaseUIController ): @web.expose def index( self, trans, **kwargs ): return trans.fill_template( "mobile/index.mako" ) - + @web.expose def history_list( self, trans ): return trans.fill_template( "mobile/history/list.mako" ) - + @web.expose def history_detail( self, trans, id ): history = trans.sa_session.query( trans.app.model.History ).get( id ) @@ -26,7 +26,7 @@ class Mobile( BaseUIController ): dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( id ) assert dataset.history.user == trans.user return trans.fill_template( "mobile/dataset/peek.mako", dataset=dataset ) - + @web.expose def settings( self, trans, email=None, password=None ): message = None @@ -38,7 +38,7 @@ class Mobile( BaseUIController ): error = self.__login( trans, email, password ) message = error or "Login changed" return trans.fill_template( "mobile/settings.mako", message=message ) - + def __logout( self, trans ): trans.log_event( "User logged out" ) trans.handle_user_logout() diff --git a/lib/galaxy/webapps/galaxy/controllers/page.py b/lib/galaxy/webapps/galaxy/controllers/page.py index dcf26ac3202..b927c27c70c 100644 --- a/lib/galaxy/webapps/galaxy/controllers/page.py +++ b/lib/galaxy/webapps/galaxy/controllers/page.py @@ -19,7 +19,7 @@ class PageListGrid( grids.Grid ): class URLColumn( grids.PublicURLColumn ): def get_value( self, trans, grid, item ): return url_for(controller='page', action='display_by_username_and_slug', username=item.user.username, slug=item.slug ) - + # Grid definition use_panels = True title = "Pages" @@ -35,10 +35,10 @@ class PageListGrid( grids.Grid ): grids.GridColumn( "Created", key="create_time", format=time_ago ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), ] - columns.append( - grids.MulticolFilterColumn( - "Search", - cols_to_filter=[ columns[0], columns[2] ], + columns.append( + grids.MulticolFilterColumn( + "Search", + cols_to_filter=[ columns[0], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) global_actions = [ @@ -53,7 +53,7 @@ class PageListGrid( grids.Grid ): ] def apply_query_filter( self, trans, query, **kwargs ): return query.filter_by( user=trans.user, deleted=False ) - + class PageAllPublishedGrid( grids.Grid ): # Grid definition use_panels = True @@ -66,14 +66,14 @@ class PageAllPublishedGrid( grids.Grid ): grids.PublicURLColumn( "Title", key="title", filterable="advanced" ), grids.OwnerAnnotationColumn( "Annotation", key="annotation", model_annotation_association_class=model.PageAnnotationAssociation, filterable="advanced" ), grids.OwnerColumn( "Owner", key="username", model_class=model.User, filterable="advanced" ), - grids.CommunityRatingColumn( "Community Rating", key="rating" ), + grids.CommunityRatingColumn( "Community Rating", key="rating" ), grids.CommunityTagsColumn( "Community Tags", key="tags", model_tag_association_class=model.PageTagAssociation, filterable="advanced", grid_name="PageAllPublishedGrid" ), grids.ReverseSortColumn( "Last Updated", key="update_time", format=time_ago ) ] - columns.append( - grids.MulticolFilterColumn( - "Search title, annotation, owner, and tags", - cols_to_filter=[ columns[0], columns[1], columns[2], columns[4] ], + columns.append( + grids.MulticolFilterColumn( + "Search title, annotation, owner, and tags", + cols_to_filter=[ columns[0], columns[1], columns[2], columns[4] ], key="free-text-search", visible=False, filterable="standard" ) ) def build_initial_query( self, trans, **kwargs ): @@ -81,7 +81,7 @@ class PageAllPublishedGrid( grids.Grid ): return trans.sa_session.query( self.model_class ).join( model.User.table ) def apply_query_filter( self, trans, query, **kwargs ): return query.filter( self.model_class.deleted==False ).filter( self.model_class.published==True ) - + class ItemSelectionGrid( grids.Grid ): """ Base class for pages' item selection grids. """ # Custom columns. @@ -95,16 +95,16 @@ class ItemSelectionGrid( grids.Grid ): # Grid definition. show_item_checkboxes = True template = "/page/select_items_grid.mako" - async_template = "/page/select_items_grid_async.mako" + async_template = "/page/select_items_grid_async.mako" default_filter = { "deleted" : "False" , "sharing" : "All" } default_sort_key = "-update_time" use_async = True use_paging = True num_rows_per_page = 10 - + def apply_query_filter( self, trans, query, **kwargs ): return query.filter_by( user=trans.user ) - + class HistorySelectionGrid( ItemSelectionGrid ): """ Grid for selecting histories. """ # Grid definition. @@ -118,16 +118,16 @@ class HistorySelectionGrid( ItemSelectionGrid ): grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ), grids.SharingStatusColumn( "Sharing", key="sharing", filterable="advanced", sortable=False, visible=False ), ] - columns.append( - grids.MulticolFilterColumn( - "Search", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( + grids.MulticolFilterColumn( + "Search", + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) - + def apply_query_filter( self, trans, query, **kwargs ): return query.filter_by( user=trans.user, purged=False ) - + class HistoryDatasetAssociationSelectionGrid( ItemSelectionGrid ): """ Grid for selecting HDAs. """ # Grid definition. @@ -141,18 +141,18 @@ class HistoryDatasetAssociationSelectionGrid( ItemSelectionGrid ): grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ), grids.SharingStatusColumn( "Sharing", key="sharing", filterable="advanced", sortable=False, visible=False ), ] - columns.append( - grids.MulticolFilterColumn( - "Search", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( + grids.MulticolFilterColumn( + "Search", + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) def apply_query_filter( self, trans, query, **kwargs ): # To filter HDAs by user, need to join HDA and History table and then filter histories by user. This is necessary because HDAs do not have # a user relation. return query.select_from( model.HistoryDatasetAssociation.table.join( model.History.table ) ).filter( model.History.user == trans.user ) - - + + class WorkflowSelectionGrid( ItemSelectionGrid ): """ Grid for selecting workflows. """ # Grid definition. @@ -166,10 +166,10 @@ class WorkflowSelectionGrid( ItemSelectionGrid ): grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ), grids.SharingStatusColumn( "Sharing", key="sharing", filterable="advanced", sortable=False, visible=False ), ] - columns.append( - grids.MulticolFilterColumn( - "Search", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( + grids.MulticolFilterColumn( + "Search", + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -186,13 +186,13 @@ class PageSelectionGrid( ItemSelectionGrid ): grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ), grids.SharingStatusColumn( "Sharing", key="sharing", filterable="advanced", sortable=False, visible=False ), ] - columns.append( - grids.MulticolFilterColumn( + columns.append( + grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1] ], + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) - + class VisualizationSelectionGrid( ItemSelectionGrid ): """ Grid for selecting visualizations. """ # Grid definition. @@ -204,36 +204,36 @@ class VisualizationSelectionGrid( ItemSelectionGrid ): grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.VisualizationTagAssociation, filterable="advanced", grid_name="VisualizationListGrid" ), grids.SharingStatusColumn( "Sharing", key="sharing", filterable="advanced", sortable=False ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), - ] - columns.append( - grids.MulticolFilterColumn( - "Search", - cols_to_filter=[ columns[0], columns[2] ], + ] + columns.append( + grids.MulticolFilterColumn( + "Search", + cols_to_filter=[ columns[0], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) - ) - + ) + class _PageContentProcessor( _BaseHTMLProcessor ): """ Processes page content to produce HTML that is suitable for display. For now, processor renders embedded objects. """ - + def __init__( self, trans, encoding, type, render_embed_html_fn ): _BaseHTMLProcessor.__init__( self, encoding, type) self.trans = trans self.ignore_content = False self.num_open_tags_for_ignore = 0 self.render_embed_html_fn = render_embed_html_fn - + def unknown_starttag( self, tag, attrs ): """ Called for each start tag; attrs is a list of (attr, value) tuples. """ - + # If ignoring content, just increment tag count and ignore. if self.ignore_content: self.num_open_tags_for_ignore += 1 return - + # Not ignoring tag; look for embedded content. embedded_item = False for attribute in attrs: - if ( attribute[0] == "class" ) and ( "embedded-item" in attribute[1].split(" ") ): + if ( attribute[0] == "class" ) and ( "embedded-item" in attribute[1].split(" ") ): embedded_item = True break # For embedded content, set ignore flag to ignore current content and add new content for embedded item. @@ -241,7 +241,7 @@ class _PageContentProcessor( _BaseHTMLProcessor ): # Set processing attributes to ignore content. self.ignore_content = True self.num_open_tags_for_ignore = 1 - + # Insert content for embedded element. for attribute in attrs: name = attribute[0] @@ -251,33 +251,33 @@ class _PageContentProcessor( _BaseHTMLProcessor ): embed_html = self.render_embed_html_fn( self.trans, item_class, item_id ) self.pieces.append( embed_html ) return - + # Default behavior: not ignoring and no embedded content. _BaseHTMLProcessor.unknown_starttag( self, tag, attrs ) - + def handle_data( self, text ): """ Called for each block of plain text. """ if self.ignore_content: return _BaseHTMLProcessor.handle_data( self, text ) - + def unknown_endtag( self, tag ): """ Called for each end tag. """ - + # If ignoring content, see if current tag is the end of content to ignore. if self.ignore_content: - self.num_open_tags_for_ignore -= 1 + self.num_open_tags_for_ignore -= 1 if self.num_open_tags_for_ignore == 0: # Done ignoring content. self.ignore_content = False return - - # Default behavior: + + # Default behavior: _BaseHTMLProcessor.unknown_endtag( self, tag ) - -class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHistoryMixin, + +class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHistoryMixin, UsesStoredWorkflowMixin, UsesVisualizationMixin, UsesItemRatings ): - + _page_list = PageListGrid() _all_published_list = PageAllPublishedGrid() _history_selection_grid = HistorySelectionGrid() @@ -285,9 +285,9 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist _datasets_selection_grid = HistoryDatasetAssociationSelectionGrid() _page_selection_grid = PageSelectionGrid() _visualization_selection_grid = VisualizationSelectionGrid() - + @web.expose - @web.require_login() + @web.require_login() def list( self, trans, *args, **kwargs ): """ List user's pages. """ # Handle operation @@ -302,10 +302,10 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist if operation == "share or publish": return self.sharing( trans, **kwargs ) session.flush() - + # Build grid HTML. grid = self._page_list( trans, *args, **kwargs ) - + # Build list of pages shared with user. shared_by_others = trans.sa_session \ .query( model.PageUserShareAssociation ) \ @@ -314,10 +314,10 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist .filter( model.Page.deleted == False ) \ .order_by( desc( model.Page.update_time ) ) \ .all() - + # Render grid wrapped in panels return trans.fill_template( "page/index.mako", grid=grid, shared_by_others=shared_by_others ) - + @web.expose def list_published( self, trans, *args, **kwargs ): grid = self._all_published_list( trans, *args, **kwargs ) @@ -327,7 +327,7 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist # Render grid wrapped in panels return trans.fill_template( "page/list_published.mako", grid=grid ) - + @web.expose @web.require_login( "create pages" ) def create( self, trans, page_title="", page_slug="", page_annotation="" ): @@ -366,7 +366,7 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist # Display the management page ## trans.set_message( "Page '%s' created" % page.title ) return trans.response.send_redirect( web.url_for(controller='page', action='list' ) ) - return trans.show_form( + return trans.show_form( web.FormBuilder( web.url_for(controller='page', action='create'), "Create new page", submit_text="Submit" ) .add_text( "page_title", "Page title", value=page_title, error=page_title_err ) .add_text( "page_slug", "Page identifier", value=page_slug, error=page_slug_err, @@ -378,7 +378,7 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist .add_text( "page_annotation", "Page annotation", value=page_annotation, error=page_annotation_err, help="A description of the page; annotation is shown alongside published pages."), template="page/create.mako" ) - + @web.expose @web.require_login( "edit pages" ) def edit( self, trans, id, page_title="", page_slug="", page_annotation="" ): @@ -417,7 +417,7 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist page_annotation = self.get_item_annotation_str( trans.sa_session, trans.user, page ) if not page_annotation: page_annotation = "" - return trans.show_form( + return trans.show_form( web.FormBuilder( web.url_for(controller='page', action='edit', id=encoded_id ), "Edit page attributes", submit_text="Submit" ) .add_text( "page_title", "Page title", value=page_title, error=page_title_err ) .add_text( "page_slug", "Page identifier", value=page_slug, error=page_slug_err, @@ -429,18 +429,18 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist .add_text( "page_annotation", "Page annotation", value=page_annotation, error=page_annotation_err, help="A description of the page; annotation is shown alongside published pages."), template="page/create.mako" ) - + @web.expose @web.require_login( "edit pages" ) def edit_content( self, trans, id ): """ - Render the main page editor interface. + Render the main page editor interface. """ id = trans.security.decode_id( id ) page = trans.sa_session.query( model.Page ).get( id ) assert page.user == trans.user return trans.fill_template( "page/editor.mako", page=page ) - + @web.expose @web.require_login( "use Galaxy pages" ) def sharing( self, trans, id, **kwargs ): @@ -476,7 +476,7 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist return trans.fill_template( "/sharing_base.mako", item=page, use_panels=True ) - + @web.expose @web.require_login( "use Galaxy pages" ) def share( self, trans, id, email="", use_panels=False ): @@ -514,24 +514,24 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist item=page, email=email, use_panels=use_panels ) - + @web.expose - @web.require_login() + @web.require_login() def save( self, trans, id, content, annotations ): id = trans.security.decode_id( id ) page = trans.sa_session.query( model.Page ).get( id ) assert page.user == trans.user - + # Sanitize content content = sanitize_html( content, 'utf-8', 'text/html' ) - + # Add a new revision to the page with the provided content. page_revision = model.PageRevision() page_revision.title = page.title page_revision.page = page page.latest_revision = page_revision page_revision.content = content - + # Save annotations. annotations = from_json_string( annotations ) for annotation_dict in annotations: @@ -541,7 +541,7 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist if not item: raise RuntimeError( "cannot find annotated item" ) text = sanitize_html( annotation_dict[ 'text' ], 'utf-8', 'text/html' ) - + # Add/update annotation. if item_id and item_class and text: # Get annotation association. @@ -564,9 +564,9 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist # Set annotation user text. annotation_assoc.annotation = text trans.sa_session.flush() - + @web.expose - @web.require_login() + @web.require_login() def display( self, trans, id ): id = trans.security.decode_id( id ) page = trans.sa_session.query( model.Page ).get( id ) @@ -576,7 +576,7 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist @web.expose def display_by_username_and_slug( self, trans, username, slug ): - """ Display page based on a username and slug. """ + """ Display page based on a username and slug. """ # Get page. session = trans.sa_session @@ -586,11 +586,11 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist raise web.httpexceptions.HTTPNotFound() # Security check raises error if user cannot access page. self.security_check( trans, page, False, True) - + # Process page content. processor = _PageContentProcessor( trans, 'utf-8', 'text/html', self._get_embed_html ) processor.feed( page.latest_revision.content ) - + # Get rating data. user_item_rating = 0 if trans.get_user(): @@ -600,13 +600,13 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist else: user_item_rating = 0 ave_item_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, page ) - + # Output is string, so convert to unicode for display. page_content = unicode( processor.output(), 'utf-8' ) - return trans.fill_template_mako( "page/display.mako", item=page, item_data=page_content, + return trans.fill_template_mako( "page/display.mako", item=page, item_data=page_content, user_item_rating = user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings, content_only=True ) - + @web.expose @web.require_login( "use Galaxy pages" ) def set_accessible_async( self, trans, id=None, accessible=False ): @@ -637,7 +637,7 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist page_rating = self.rate_item( trans.sa_session, trans.get_user(), page, rating ) return self.get_ave_item_rating_data( trans.sa_session, page ) - + @web.expose def get_embed_html_async( self, trans, id ): """ Returns HTML for embedding a workflow in a page. """ @@ -658,57 +658,57 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist trans.sa_session.flush() return_dict = { "name" : page.title, "link" : url_for(controller='page', action="display_by_username_and_slug", username=page.user.username, slug=page.slug ) } return return_dict - + @web.expose @web.require_login("select a history from saved histories") def list_histories_for_selection( self, trans, **kwargs ): """ Returns HTML that enables a user to select one or more histories. """ # Render the list view return self._history_selection_grid( trans, **kwargs ) - + @web.expose @web.require_login("select a workflow from saved workflows") def list_workflows_for_selection( self, trans, **kwargs ): """ Returns HTML that enables a user to select one or more workflows. """ # Render the list view return self._workflow_selection_grid( trans, **kwargs ) - + @web.expose @web.require_login("select a visualization from saved visualizations") def list_visualizations_for_selection( self, trans, **kwargs ): """ Returns HTML that enables a user to select one or more visualizations. """ # Render the list view return self._visualization_selection_grid( trans, **kwargs ) - + @web.expose @web.require_login("select a page from saved pages") def list_pages_for_selection( self, trans, **kwargs ): """ Returns HTML that enables a user to select one or more pages. """ # Render the list view return self._page_selection_grid( trans, **kwargs ) - + @web.expose @web.require_login("select a dataset from saved datasets") def list_datasets_for_selection( self, trans, **kwargs ): """ Returns HTML that enables a user to select one or more datasets. """ # Render the list view return self._datasets_selection_grid( trans, **kwargs ) - + @web.expose @web.require_login("get annotation table for history") def get_history_annotation_table( self, trans, id ): """ Returns HTML for an annotation table for a history. """ history = self.get_history( trans, id, False, True ) - + if history: datasets = self.get_history_datasets( trans, history ) return trans.fill_template( "page/history_annotation_table.mako", history=history, datasets=datasets, show_deleted=False ) - + @web.expose def get_editor_iframe( self, trans ): """ Returns the document for the page editor's iframe. """ return trans.fill_template( "page/wymiframe.mako" ) - + def get_page( self, trans, id, check_ownership=True, check_accessible=False ): """Get a page from the database by id.""" # Load history from database @@ -718,10 +718,10 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist error( "Page not found" ) else: return self.security_check( trans, page, check_ownership, check_accessible ) - + def get_item( self, trans, id ): return self.get_page( trans, id ) - + def _get_embed_html( self, trans, item_class, item_id ): """ Returns HTML for embedding an item in a page. """ item_class = self.get_class( item_class ) @@ -748,9 +748,9 @@ class PageController( BaseUIController, SharableMixin, UsesAnnotations, UsesHist visualization.annotation = self.get_item_annotation_str( trans.sa_session, visualization.user, visualization ) if visualization: return trans.fill_template( "visualization/embed.mako", item=visualization, item_data=None ) - + elif item_class == model.Page: pass - - - + + + diff --git a/lib/galaxy/webapps/galaxy/controllers/request_type.py b/lib/galaxy/webapps/galaxy/controllers/request_type.py index 8a14bd42968..944e44e805c 100644 --- a/lib/galaxy/webapps/galaxy/controllers/request_type.py +++ b/lib/galaxy/webapps/galaxy/controllers/request_type.py @@ -39,7 +39,7 @@ class RequestTypeGrid( grids.Grid ): use_paging = True default_filter = dict( deleted="False" ) columns = [ - NameColumn( "Name", + NameColumn( "Name", key="name", link=( lambda item: iff( item.deleted, None, dict( operation="view_request_type", id=item.id ) ) ), attach_popup=True, @@ -47,18 +47,18 @@ class RequestTypeGrid( grids.Grid ): DescriptionColumn( "Description", key='desc', filterable="advanced" ), - RequestFormColumn( "Request Form", + RequestFormColumn( "Request Form", link=( lambda item: iff( item.deleted, None, dict( operation="view_form_definition", id=item.request_form.id ) ) ) ), - SampleFormColumn( "Sample Form", + SampleFormColumn( "Sample Form", link=( lambda item: iff( item.deleted, None, dict( operation="view_form_definition", id=item.sample_form.id ) ) ) ), ExternalServiceColumn( "External Services" ), - grids.DeletedColumn( "Deleted", - key="deleted", - visible=False, + grids.DeletedColumn( "Deleted", + key="deleted", + visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -67,7 +67,7 @@ class RequestTypeGrid( grids.Grid ): grids.GridOperation( "Edit permissions", allow_multiple=False, condition=( lambda item: not item.deleted ) ), grids.GridOperation( "Use run details template", allow_multiple=False, condition=( lambda item: not item.deleted and not item.run_details ) ), grids.GridOperation( "Delete", allow_multiple=True, condition=( lambda item: not item.deleted ) ), - grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), + grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ), ] global_actions = [ grids.GridAction( "Create new request type", dict( controller='request_type', action='create_request_type' ) ) @@ -117,9 +117,9 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): external_service_select_fields_list = [] # get all the external services selected till now external_services_list = self.__get_external_services( trans, **kwd ) - for index, external_service in enumerate( external_services_list ): - external_service_select_field = self.__build_external_service_select_field( trans, - 'external_service_id_%i' % index, + for index, external_service in enumerate( external_services_list ): + external_service_select_field = self.__build_external_service_select_field( trans, + 'external_service_id_%i' % index, external_service ) external_service_select_fields_list.append( external_service_select_field ) if params.get( 'add_state_button', False ): @@ -132,7 +132,7 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): del rt_states_widgets[ index-1 ] elif params.get( 'add_external_service_button', False ): # create a new one - external_service_select_field = self.__build_external_service_select_field( trans, + external_service_select_field = self.__build_external_service_select_field( trans, 'external_service_id_%i' % len( external_services_list ) ) external_service_select_fields_list.append( external_service_select_field ) elif params.get( 'create_request_type_button', False ): @@ -184,9 +184,9 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): widget_fields_have_contents = self.widget_fields_have_contents( widgets ) # get all the external services selected till now external_service_select_fields_list = [] - for index, external_service in enumerate( request_type.external_services ): - external_service_select_field = self.__build_external_service_select_field( trans, - 'external_service_id_%i' % index, + for index, external_service in enumerate( request_type.external_services ): + external_service_select_field = self.__build_external_service_select_field( trans, + 'external_service_id_%i' % index, external_service ) external_service_select_fields_list.append( external_service_select_field ) return trans.fill_template( '/admin/request_type/edit_request_type.mako', @@ -222,9 +222,9 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): index = int( kwd[ 'remove_external_service_button' ].split(' ')[3] ) - 1 del external_services_list[index] #if external_services_list - for index, external_service in enumerate( external_services_list ): - external_service_select_field = self.__build_external_service_select_field( trans, - 'external_service_id_%i' % index, + for index, external_service in enumerate( external_services_list ): + external_service_select_field = self.__build_external_service_select_field( trans, + 'external_service_id_%i' % index, external_service ) external_service_select_fields_list.append( external_service_select_field ) return trans.fill_template( '/admin/request_type/edit_request_type.mako', @@ -255,7 +255,7 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): request_form = trans.sa_session.query( trans.model.FormDefinition ).get( trans.security.decode_id( request_form_id ) ) except: return invalid_id_redirect( trans, 'request_type', request_type_id, 'form definition', action='browse_request_types' ) - + try: sample_form = trans.sa_session.query( trans.model.FormDefinition ).get( trans.security.decode_id( sample_form_id ) ) except: @@ -282,7 +282,7 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): request_type = trans.model.RequestType( name=name, desc=desc, request_form=request_form, - sample_form=sample_form ) + sample_form=sample_form ) trans.sa_session.add( request_type ) trans.sa_session.flush() i = 0 @@ -290,7 +290,7 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): if kwd.has_key( 'state_name_%i' % i ): name = util.restore_text( params.get( 'state_name_%i' % i, '' ) ) desc = util.restore_text( params.get( 'state_desc_%i' % i, '' ) ) - sample_state = trans.model.SampleState( name, desc, request_type ) + sample_state = trans.model.SampleState( name, desc, request_type ) trans.sa_session.add( sample_state ) trans.sa_session.flush() i += 1 @@ -304,10 +304,10 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): request_type.add_external_service_association( trans, external_service ) return request_type def __get_populated_request_type_widgets( self, trans, **kwd ): - request_form_definitions = self.get_all_forms( trans, + request_form_definitions = self.get_all_forms( trans, filter=dict( deleted=False ), form_type=trans.model.FormDefinition.types.REQUEST ) - sample_form_definitions = self.get_all_forms( trans, + sample_form_definitions = self.get_all_forms( trans, filter=dict( deleted=False ), form_type=trans.model.FormDefinition.types.SAMPLE ) if not request_form_definitions or not sample_form_definitions: @@ -327,9 +327,9 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): select_field_name='sample_form_id', selected_value=sample_form_id, refresh_on_change=False ) - rt_info_widgets = [ dict( label='Name', + rt_info_widgets = [ dict( label='Name', widget=TextField( 'name', 40, util.restore_text( params.get( 'name', '' ) ) ) ), - dict( label='Description', + dict( label='Description', widget=TextField( 'desc', 40, util.restore_text( params.get( 'desc', '' ) ) ) ), dict( label='Request form', widget=request_form_id_select_field ), @@ -340,13 +340,13 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): i=0 while True: if kwd.has_key( 'state_name_%i' % i ): - rt_states.append( ( util.restore_text( params.get( 'state_name_%i' % i, '' ) ), + rt_states.append( ( util.restore_text( params.get( 'state_name_%i' % i, '' ) ), util.restore_text( params.get( 'state_desc_%i' % i, '' ) ) ) ) i += 1 else: break return rt_info_widgets, rt_states - + @web.expose @web.require_admin def view_request_type( self, trans, **kwd ): @@ -361,7 +361,7 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): # See if we have any associated templates widgets = request_type.get_template_widgets( trans ) widget_fields_have_contents = self.widget_fields_have_contents( widgets ) - return trans.fill_template( '/admin/request_type/view_request_type.mako', + return trans.fill_template( '/admin/request_type/view_request_type.mako', request_type=request_type, widgets=widgets, widget_fields_have_contents=widget_fields_have_contents, @@ -446,7 +446,7 @@ class RequestType( BaseUIController, UsesFormDefinitionsMixin ): # ===== Methods for building SelectFields used on various admin_requests forms def __build_external_service_select_field( self, trans, select_field_name, external_service=None ): if external_service: - selected_value = trans.security.encode_id( external_service.id ) + selected_value = trans.security.encode_id( external_service.id ) else: selected_value = 'none' all_external_services = trans.sa_session.query( trans.model.ExternalService ).filter( trans.model.ExternalService.table.c.deleted==False ).all() diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py index 23295a90124..3e45682dfac 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_admin.py @@ -27,7 +27,7 @@ class AdminRequestsGrid( RequestsGrid ): operations.append( grids.GridOperation( "Undelete", condition=( lambda item: item.deleted ) ) ) global_actions = [ grids.GridAction( "Create new request", dict( controller='requests_common', - action='create_request', + action='create_request', cntrller='requests_admin' ) ) ] @@ -57,14 +57,14 @@ class DataTransferGrid( grids.Grid ): preserve_state = True use_paging = False columns = [ - NameColumn( "Name", + NameColumn( "Name", link=( lambda item: dict( operation="view", id=item.id ) ), attach_popup=True, filterable="advanced" ), SizeColumn( "Size", filterable="advanced" ), - grids.GridColumn( "Last Updated", - key="update_time", + grids.GridColumn( "Last Updated", + key="update_time", format=time_ago ), ExternalServiceColumn( 'External service', link=( lambda item: dict( operation="view_external_service", id=item.external_service.id ) ), ), @@ -72,8 +72,8 @@ class DataTransferGrid( grids.Grid ): filterable="advanced", label_id_prefix='datasetTransferStatus-' ), ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -172,7 +172,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): if not comment: status='error' message='A reason for rejecting the request is required.' - return trans.fill_template( '/admin/requests/reject.mako', + return trans.fill_template( '/admin/requests/reject.mako', cntrller='requests_admin', request=request, status=status, @@ -268,7 +268,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): sample_id=trans.security.encode_id( selected_sample_datasets[0].sample.id ), status=status, message=message ) ) - return trans.fill_template( '/admin/requests/rename_datasets.mako', + return trans.fill_template( '/admin/requests/rename_datasets.mako', sample=selected_sample_datasets[0].sample, id_list=id_list ) elif operation == "transfer": @@ -284,13 +284,13 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): request_id = trans.security.encode_id( sample.request.id ) library_id = trans.security.encode_id( sample.library.id ) self.datatx_grid.title = 'Manage "%s" datasets' % sample.name - self.datatx_grid.global_actions = [ grids.GridAction( "Browse target data library", - dict( controller='library_common', - action='browse_library', - cntrller='library_admin', + self.datatx_grid.global_actions = [ grids.GridAction( "Browse target data library", + dict( controller='library_common', + action='browse_library', + cntrller='library_admin', id=library_id ) ), - grids.GridAction( "Browse this request", - dict( controller='requests_common', + grids.GridAction( "Browse this request", + dict( controller='requests_common', action='view_request', cntrller='requests_admin', id=request_id ) ) ] @@ -329,7 +329,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): new_name = util.sanitize_for_filename( new_name ) if selected_option == 'none': sample_dataset.name = new_name - else: + else: sample_dataset.name = '%s_%s' % ( selected_option, new_name ) trans.sa_session.add( sample_dataset ) trans.sa_session.flush() @@ -341,7 +341,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): message = 'Changes saved successfully. The following datasets were renamed incorrectly: %s.' % str( incorrect_dataset_names ) else: message = 'Changes saved successfully.' - return trans.fill_template( '/admin/requests/rename_datasets.mako', + return trans.fill_template( '/admin/requests/rename_datasets.mako', sample=sample, id_list=id_list, message=message, @@ -397,7 +397,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): id=trans.security.encode_id( request.id ), status=status, message=message ) ) - # Save the sample datasets + # Save the sample datasets sample_dataset_file_names = self.__create_sample_datasets( trans, sample, selected_datasets_to_transfer, external_service ) if sample_dataset_file_names: message = 'Datasets (%s) have been selected for sample (%s)' % \ @@ -437,7 +437,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): output = pexpect.run( cmd, events={ '\(yes\/no\)\.*' : 'yes\r\n', '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, + pexpect.TIMEOUT : print_ticks }, timeout=10 ) for password_str in [ 'Password:\r\n', 'password:\r\n' ]: # Eliminate the output created using ssh from the tree @@ -482,9 +482,9 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): # Handle the authentication message if keys are not set - the message is # something like: "Are you sure you want to continue connecting (yes/no)." output = pexpect.run( cmd, - events={ '\(yes\/no\)\.*' : 'yes\r\n', + events={ '\(yes\/no\)\.*' : 'yes\r\n', '.ssword:*' : scp_configs[ 'password' ] + '\r\n', - pexpect.TIMEOUT : print_ticks }, + pexpect.TIMEOUT : print_ticks }, timeout=10 ) if 'No such file or directory' in output: status = 'error' @@ -528,7 +528,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): def __rename_dataset( self, sample, filepath, scp_configs ): name = filepath.split( '/' )[-1] options = sample.request.type.rename_dataset_options - option = scp_configs.get( 'rename_dataset', options.NO ) + option = scp_configs.get( 'rename_dataset', options.NO ) if option == options.SAMPLE_NAME: new_name = sample.name + '_' + name if option == options.EXPERIMENT_AND_SAMPLE_NAME: @@ -537,7 +537,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): new_name = sample.request.name + '_' + name else: new_name = name - return util.sanitize_for_filename( new_name ) + return util.sanitize_for_filename( new_name ) def __ensure_library_add_permission( self, trans, target_library, target_folder ): """ Ensures the current admin user has ADD_LIBRARY permission on the target data library and folder. @@ -547,13 +547,13 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): flush_needed = False if not trans.app.security_agent.can_add_library_item( current_user_roles, target_library ): lp = trans.model.LibraryPermissions( trans.app.security_agent.permitted_actions.LIBRARY_ADD.action, - target_library, + target_library, current_user_private_role ) trans.sa_session.add( lp ) flush_needed = True if not trans.app.security_agent.can_add_library_item( current_user_roles, target_folder ): lfp = trans.model.LibraryFolderPermissions( trans.app.security_agent.permitted_actions.LIBRARY_ADD.action, - target_folder, + target_folder, current_user_private_role ) trans.sa_session.add( lfp ) flush_needed = True @@ -585,7 +585,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): %(FILE)s ''' # Here we group all the sample_datasets by the external service used to transfer them. - # The idea is to bundle up the sample_datasets which uses the same external service and + # The idea is to bundle up the sample_datasets which uses the same external service and # send a single AMQP message to the galaxy_listener dataset_elements = {} for sample_dataset in selected_sample_datasets: @@ -600,7 +600,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): sample_dataset.status = trans.app.model.SampleDataset.transfer_status.IN_QUEUE trans.sa_session.add( sample_dataset ) trans.sa_session.flush() - # Finally prepend the external service info to the sets of sample datasets + # Finally prepend the external service info to the sets of sample datasets messages = [] for external_service, dataset_elem in dataset_elements.items(): external_service.load_data_transfer_settings( trans ) @@ -685,22 +685,22 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): external_service_type=external_service_type ) else: # TODO: Using RabbitMq for now, but eliminate this entire block when we replace RabbitMq with Galaxy's - # own messaging engine. We're holding off on using the new way to transfer files manually until we + # own messaging engine. We're holding off on using the new way to transfer files manually until we # implement a Galaxy-proprietary messaging engine because the deferred job plugins currently perform # constant db hits to check for deferred jobs that are not in a finished state. # Create the message messages = self.__create_data_transfer_messages( trans, sample, sample_datasets ) - # Send the messages + # Send the messages for rmq_msg in messages: try: - conn = amqp.Connection( host=trans.app.config.amqp[ 'host' ] + ":" + trans.app.config.amqp[ 'port' ], - userid=trans.app.config.amqp[ 'userid' ], - password=trans.app.config.amqp[ 'password' ], - virtual_host=trans.app.config.amqp[ 'virtual_host' ], - insist=False ) + conn = amqp.Connection( host=trans.app.config.amqp[ 'host' ] + ":" + trans.app.config.amqp[ 'port' ], + userid=trans.app.config.amqp[ 'userid' ], + password=trans.app.config.amqp[ 'password' ], + virtual_host=trans.app.config.amqp[ 'virtual_host' ], + insist=False ) chan = conn.channel() - msg = amqp.Message( rmq_msg, - content_type='text/plain', + msg = amqp.Message( rmq_msg, + content_type='text/plain', application_headers={ 'msg_type': 'data_transfer' } ) msg.properties[ "delivery_mode" ] = 2 chan.basic_publish( msg, @@ -722,7 +722,7 @@ class RequestsAdmin( BaseUIController, UsesFormDefinitionsMixin ): @web.expose def update_sample_dataset_status(self, trans, cntrller, sample_dataset_ids, new_status, error_msg=None ): # check if the new status is a valid transfer status - possible_status_list = [ v[1] for v in trans.app.model.SampleDataset.transfer_status.items() ] + possible_status_list = [ v[1] for v in trans.app.model.SampleDataset.transfer_status.items() ] if new_status not in possible_status_list: trans.response.status = 400 return 400, "The requested transfer status ( %s ) is not a valid transfer status." % new_status diff --git a/lib/galaxy/webapps/galaxy/controllers/requests_common.py b/lib/galaxy/webapps/galaxy/controllers/requests_common.py index 95b2d36d07a..211140be046 100644 --- a/lib/galaxy/webapps/galaxy/controllers/requests_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/requests_common.py @@ -49,7 +49,7 @@ class RequestsGrid( grids.Grid ): .filter( model.RequestEvent.table.c.id.in_( select( columns=[ func.max( model.RequestEvent.table.c.id ) ], from_obj=model.RequestEvent.table, group_by=model.RequestEvent.table.c.request_id ) ) ) - + # Grid definition title = "Sequencing Requests" template = "requests/grid.mako" @@ -59,31 +59,31 @@ class RequestsGrid( grids.Grid ): use_paging = True default_filter = dict( state="All", deleted="False" ) columns = [ - NameColumn( "Name", - key="name", + NameColumn( "Name", + key="name", link=( lambda item: dict( operation="view_request", id=item.id ) ), - attach_popup=True, + attach_popup=True, filterable="advanced" ), DescriptionColumn( "Description", key='desc', filterable="advanced" ), - SamplesColumn( "Samples", + SamplesColumn( "Samples", link=( lambda item: iff( item.deleted, None, dict( operation="edit_samples", id=item.id ) ) ) ), TypeColumn( "Type", link=( lambda item: iff( item.deleted, None, dict( operation="view_type", id=item.type.id ) ) ) ), grids.GridColumn( "Last Updated", key="update_time", format=time_ago ), - grids.DeletedColumn( "Deleted", - key="deleted", - visible=False, + grids.DeletedColumn( "Deleted", + key="deleted", + visible=False, filterable="advanced" ), - StateColumn( "State", + StateColumn( "State", key='state', filterable="advanced", link=( lambda item: iff( item.deleted, None, dict( operation="view_request_history", id=item.id ) ) ) ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -176,13 +176,13 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): user = trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( user_id ) ) except TypeError, e: # We must have an email address rather than an encoded user id - # This is because the galaxy.base.js creates a search+select box + # This is because the galaxy.base.js creates a search+select box # when there are more than 20 items in a SelectField. user = trans.sa_session.query( trans.model.User ) \ .filter( trans.model.User.table.c.email==util.restore_text( user_id ) ) \ .first() user_id_encoded = False - + elif not is_admin: user = trans.user else: @@ -195,8 +195,8 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): elif user is None: message = 'Invalid user ID (%s)' % str(user_id) status = 'error' - # when creating a request from the user perspective, check if the - # user has access permission to this request_type + # when creating a request from the user perspective, check if the + # user has access permission to this request_type elif cntrller == 'requests' and not trans.app.security_agent.can_access_request_type( user.all_roles(), request_type ): message = '%s does not have access permission to the "%s" request type.' % ( user.email, request_type.name ) status = 'error' @@ -219,11 +219,11 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): widgets = [] if request_type is not None or status == 'error': # Either the user selected a request_type or an error exists on the form. - widgets.append( dict( label='Name of the Experiment', - widget=TextField( 'name', 40, util.restore_text( params.get( 'name', '' ) ) ), + widgets.append( dict( label='Name of the Experiment', + widget=TextField( 'name', 40, util.restore_text( params.get( 'name', '' ) ) ), helptext='(Required)') ) - widgets.append( dict( label='Description', - widget=TextField( 'desc', 40, util.restore_text( params.get( 'desc', '' ) )), + widgets.append( dict( label='Description', + widget=TextField( 'desc', 40, util.restore_text( params.get( 'desc', '' ) )), helptext='(Optional)') ) if request_type is not None: widgets += request_type.request_form.get_widgets( user, **kwd ) @@ -244,7 +244,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): return trans.fill_template( '/requests/common/create_request.mako', cntrller=cntrller, request_type_select_field=request_type_select_field, - request_type_select_field_selected=request_type_id, + request_type_select_field_selected=request_type_id, widgets=widgets, message=message, status=status ) @@ -265,7 +265,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): displayable_sample_widgets = self.__get_sample_widgets( trans, request, request.samples, **kwd ) request_widgets = self.__get_request_widgets( trans, request.id ) return trans.fill_template( '/requests/common/view_request.mako', - cntrller=cntrller, + cntrller=cntrller, request=request, request_widgets=request_widgets, displayable_sample_widgets=displayable_sample_widgets, @@ -293,11 +293,11 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): message = 'The changes made to request (%s) have been saved.' % request.name # Widgets to be rendered on the request form widgets = [] - widgets.append( dict( label='Name', - widget=TextField( 'name', 40, request.name ), + widgets.append( dict( label='Name', + widget=TextField( 'name', 40, request.name ), helptext='(Required)' ) ) - widgets.append( dict( label='Description', - widget=TextField( 'desc', 40, request.desc ), + widgets.append( dict( label='Description', + widget=TextField( 'desc', 40, request.desc ), helptext='(Optional)' ) ) widgets = widgets + request.type.request_form.get_widgets( request.user, request.values.content, **kwd ) # In case there is an error on the form, make sure to populate widget fields with anything the user @@ -312,7 +312,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): status=status ) def __save_request( self, trans, cntrller, request=None, **kwd ): """ - Saves changes to an existing request, or creates a new + Saves changes to an existing request, or creates a new request if received request is None. """ params = util.Params( kwd ) @@ -411,7 +411,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): trans.sa_session.flush() request.send_email_notification( trans, initial_sample_state_after_request_submitted ) message = 'The sequencing request has been submitted.' - # show the request page after submitting the request + # show the request page after submitting the request return trans.response.send_redirect( web.url_for( controller='requests_common', action='view_request', cntrller=cntrller, @@ -473,7 +473,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): samples.append( None ) # The __save_samples method requires sample_widgets, not sample objects, so we'll get what we # need by calling __get_sample_widgets(). However, we need to take care here because __get_sample_widgets() - # is used to populate the sample widget dicts from kwd, and the method assumes that a None object in the + # is used to populate the sample widget dicts from kwd, and the method assumes that a None object in the # received list of samples should be populated from the db. Since we're just re-using the method here to # change our list of samples into a list of sample widgets, we'll need to make sure to keep track of our # None objects. @@ -498,12 +498,12 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): sample_state_id = params.get( 'sample_state_id', None ) sample_state_id_select_field = self.__build_sample_state_id_select_field( trans, request, sample_state_id ) return trans.fill_template( '/requests/common/edit_samples.mako', - cntrller=cntrller, + cntrller=cntrller, request=request, encoded_selected_sample_ids=encoded_selected_sample_ids, request_widgets=request_widgets, displayable_sample_widgets=displayable_sample_widgets, - sample_copy_select_field=sample_copy_select_field, + sample_copy_select_field=sample_copy_select_field, libraries=libraries, sample_operation_select_field=sample_operation_select_field, libraries_select_field=libraries_select_field, @@ -528,7 +528,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): trans.sa_session.add( event ) trans.sa_session.flush() if cntrller == 'api': - return 200, 'Done' + return 200, 'Done' @web.expose @web.require_login( "delete sequencing requests" ) def delete_request( self, trans, cntrller, **kwd ): @@ -614,7 +614,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) ) except: return invalid_id_redirect( trans, cntrller, request_id ) - return trans.fill_template( '/requests/common/view_request_history.mako', + return trans.fill_template( '/requests/common/view_request_history.mako', cntrller=cntrller, request=request ) @web.expose @@ -657,7 +657,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): message += err_msg else: request.notification = dict( email=email_addresses, - sample_states=checked_sample_states, + sample_states=checked_sample_states, body='', subject='' ) else: @@ -796,7 +796,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): search_type = build_select_field( trans, types, 'self', 'search_type', selected_value=selected_value, refresh_on_change=False ) # Build the search_box TextField search_box = TextField( 'search_box', 50, kwd.get('search_box', '' ) ) - return trans.fill_template( '/requests/common/find_samples.mako', + return trans.fill_template( '/requests/common/find_samples.mako', cntrller=cntrller, request_states=request_states, samples=samples_list, @@ -814,7 +814,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) ) except: return invalid_id_redirect( trans, cntrller, sample_id, 'sample' ) - return trans.fill_template( '/requests/common/view_sample_history.mako', + return trans.fill_template( '/requests/common/view_sample_history.mako', cntrller=cntrller, sample=sample ) @web.expose @@ -850,11 +850,11 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): None, **kwd ) return trans.fill_template( '/requests/common/add_samples.mako', - cntrller=cntrller, + cntrller=cntrller, request=request, request_widgets=request_widgets, displayable_sample_widgets=displayable_sample_widgets, - sample_copy_select_field=sample_copy_select_field, + sample_copy_select_field=sample_copy_select_field, libraries=libraries, libraries_select_field=libraries_select_field, folders_select_field=folders_select_field, @@ -907,10 +907,10 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): field_values[ field[ 'name' ] ] = '' # Build the library_select_field and folder_select_field for the new sample being added. library_select_field, folder_select_field = self.__build_library_and_folder_select_fields( trans, - user=request.user, - sample_index=len( displayable_sample_widgets ), + user=request.user, + sample_index=len( displayable_sample_widgets ), libraries=libraries, - sample=None, + sample=None, library_id=library_id, folder_id=folder_id, **kwd ) @@ -947,7 +947,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): request=request, request_widgets=request_widgets, displayable_sample_widgets=displayable_sample_widgets, - sample_copy_select_field=sample_copy_select_field, + sample_copy_select_field=sample_copy_select_field, message=message, status=status ) @web.expose @@ -970,7 +970,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): else: external_services = None return trans.fill_template( '/requests/common/view_sample.mako', - cntrller=cntrller, + cntrller=cntrller, sample=sample, widgets=widgets, widget_fields_have_contents=widget_fields_have_contents, @@ -1049,7 +1049,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): elif transfer_status == trans.model.SampleDataset.transfer_status.ERROR: title = 'Datasets of "%s" that resulted in a transfer error' % sample.name sample_datasets = sample.transfer_error_dataset_files - return trans.fill_template( '/requests/common/view_sample_datasets.mako', + return trans.fill_template( '/requests/common/view_sample_datasets.mako', cntrller=cntrller, title=title, external_service=external_service, @@ -1229,7 +1229,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): for sample_index in range( len( sample_widgets ) ): current_sample = sample_widgets[ sample_index ] if current_sample is None: - # We have a None value because the user did not select this sample + # We have a None value because the user did not select this sample # on which to perform the action. continue request_sample = request.samples[ sample_index ] @@ -1289,7 +1289,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): trans.sa_session.add( sample ) trans.sa_session.flush() new_samples.append( sample ) - # If this sample is added when the request is already submitted then these new samples + # If this sample is added when the request is already submitted then these new samples # should be in the first sample state when saved if request.is_submitted: initial_sample_state_after_request_submitted = request.type.states[0] @@ -1348,20 +1348,20 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): # If the sample's associated SampleState is still the initial state # configured by the admin for the request's RequestType, this must be # the first time a bar code was added to the sample, so change it's state - # to the next associated SampleState. + # to the next associated SampleState. if sample.state.id == request.type.states[0].id: - # Change the sample state only if its request_type + # Change the sample state only if its request_type # has at least 2 states if len( request.type.states ) >= 2: next_sample_state = request.type.states[1] else: next_sample_state = request.type.states[0] - event = trans.model.SampleEvent( sample, - next_sample_state, + event = trans.model.SampleEvent( sample, + next_sample_state, 'Bar code associated with the sample' ) trans.sa_session.add( event ) trans.sa_session.flush() - # Next step is to update the request event history if bar codes + # Next step is to update the request event history if bar codes # have been assigned to all the samples of this request common_state = request.samples_have_common_state if request.is_submitted and common_state and len( request.type.states ) >= 2: @@ -1378,7 +1378,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): sample.workflow = sample_widget[ 'workflow_dict' ] form_values.content = sample_widget[ 'field_values' ] trans.sa_session.add_all( ( sample, form_values ) ) - trans.sa_session.flush() + trans.sa_session.flush() def __get_library_and_folder( self, trans, library_id, folder_id ): try: library = trans.sa_session.query( trans.model.Library ).get( trans.security.decode_id( library_id ) ) @@ -1413,7 +1413,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): """Return all of the active folders for the received library""" active_folders_list.extend( folder.active_folders ) for sub_folder in folder.active_folders: - self.__get_active_folders( sub_folder, active_folders_list ) + self.__get_active_folders( sub_folder, active_folders_list ) return active_folders_list # ===== Methods for handling form definition widgets ===== def __get_request_widgets( self, trans, id ): @@ -1436,7 +1436,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): request_widgets.append( dict( label=field[ 'label' ], value=None, helptext=field[ 'helptext' ] + ' (' + required_label + ')' ) ) - else: + else: request_widgets.append( dict( label=field[ 'label' ], value=field_value, helptext=field[ 'helptext' ] + ' (' + required_label + ')' ) ) @@ -1523,7 +1523,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): input_value = params.get( 'sample_%i_field_%i' % ( index, field_index ), sample.values.content[ field_name ] ) if field['type'] == CheckboxField.__name__: field_value = CheckboxField.is_checked( input_value ) - else: + else: field_value = util.restore_text( input_value ) field_values[ field_name ] = field_value library_select_field, folder_select_field = self.__build_library_and_folder_select_fields( trans=trans, @@ -1600,7 +1600,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): input_value = params.get( 'sample_%i_field_%i' % ( index, field_index ), '' ) if field['type'] == CheckboxField.__name__: field_value = CheckboxField.is_checked( input_value ) - else: + else: field_value = util.restore_text( input_value ) field_values[ field_name ] = field_value library_select_field, folder_select_field = self.__build_library_and_folder_select_fields( trans=trans, @@ -1640,14 +1640,14 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): library_select_field=library_select_field, folder_select_field=folder_select_field ) ) index += 1 - return sample_widgets + return sample_widgets # ===== Methods for building SelectFields used on various request forms ===== def __build_copy_sample_select_field( self, trans, displayable_sample_widgets ): copy_sample_index_select_field = SelectField( 'copy_sample_index' ) - copy_sample_index_select_field.add_option( 'None', -1, selected=True ) + copy_sample_index_select_field.add_option( 'None', -1, selected=True ) for index, sample_dict in enumerate( displayable_sample_widgets ): copy_sample_index_select_field.add_option( sample_dict[ 'name' ], index ) - return copy_sample_index_select_field + return copy_sample_index_select_field def __build_request_type_id_select_field( self, trans, selected_value='none' ): accessible_request_types = trans.app.security_agent.get_accessible_request_types( trans, trans.user ) return build_select_field( trans, accessible_request_types, 'name', 'request_type_id', selected_value=selected_value, refresh_on_change=True ) @@ -1707,7 +1707,7 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): # Build the sample_%i_library_id SelectField with refresh on change enabled library_select_field = build_select_field( trans, libraries, - 'name', + 'name', library_select_field_name, initial_value='none', selected_value=str( library_id ).lower(), @@ -1731,12 +1731,12 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): break folder_select_field = build_select_field( trans, folders, - 'name', + 'name', folder_select_field_name, initial_value='none', selected_value=selected_folder_id ) return library_select_field, folder_select_field - + def __build_history_select_field(self, trans, user, sample_index, sample = None, history_id=None, **kwd): params = util.Params( kwd ) history_select_field_name= "sample_%i_history_id" % sample_index @@ -1902,8 +1902,8 @@ class RequestsCommon( BaseUIController, UsesFormDefinitionsMixin ): for sample_with_bar_code in trans.sa_session.query( trans.model.Sample ) \ .filter( trans.model.Sample.table.c.bar_code == bar_code ): if sample_with_bar_code and sample_with_bar_code.id != sample.id: - message = '''The bar code (%s) associated with the sample (%s) belongs to another sample. - Bar codes must be unique across all samples, so use a different bar code + message = '''The bar code (%s) associated with the sample (%s) belongs to another sample. + Bar codes must be unique across all samples, so use a different bar code for this sample.''' % ( bar_code, sample.name ) unique = False break diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index aa5808f6d9b..7a59e035440 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -21,13 +21,13 @@ log = logging.getLogger( __name__ ) class RootController( BaseUIController, UsesHistoryMixin, UsesHistoryDatasetAssociationMixin, UsesAnnotations ): """Controller class that maps to the url root of Galaxy (i.e. '/').""" - + @web.expose def default(self, trans, target1=None, target2=None, **kwd): """Called on any url that does not match a controller method. """ raise HTTPNotFound( 'This link may not be followed from within Galaxy.' ) - + @web.expose def index(self, trans, id=None, tool_id=None, mode=None, workflow_id=None, m_c=None, m_a=None, **kwd): """Called on the root url to display the main Galaxy page. @@ -37,7 +37,7 @@ class RootController( BaseUIController, UsesHistoryMixin, UsesHistoryDatasetAsso workflow_id=workflow_id, m_c=m_c, m_a=m_a, params=kwd ) - + ## ---- Tool related ----------------------------------------------------- @web.json diff --git a/lib/galaxy/webapps/galaxy/controllers/tag.py b/lib/galaxy/webapps/galaxy/controllers/tag.py index dc1b7417acb..8e542f6fdab 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tag.py +++ b/lib/galaxy/webapps/galaxy/controllers/tag.py @@ -70,7 +70,7 @@ class TagsController ( BaseUIController, UsesTagsMixin ): """ Apply a new set of tags to an item; previous tags are deleted. """ - # Apply tags. + # Apply tags. item = self._get_item( trans, item_class, trans.security.decode_id( item_id ) ) user = trans.user self.get_tag_handler( trans ).delete_item_tags( trans, item ) @@ -108,7 +108,7 @@ class TagsController ( BaseUIController, UsesTagsMixin ): item_class = item.__class__ item_tag_assoc_class = self.get_tag_handler( trans ).get_tag_assoc_class( item_class ) # Build select statement. - cols_to_select = [ item_tag_assoc_class.table.c.tag_id, func.count( '*' ) ] + cols_to_select = [ item_tag_assoc_class.table.c.tag_id, func.count( '*' ) ] from_obj = item_tag_assoc_class.table.join( item_class.table ).join( trans.app.model.Tag.table ) where_clause = and_( trans.app.model.Tag.table.c.name.like( q + "%" ), item_tag_assoc_class.table.c.user_id == user.id ) @@ -126,7 +126,7 @@ class TagsController ( BaseUIController, UsesTagsMixin ): ac_data = "#Header|Your Tags\n" for row in result_set: tag = self.get_tag_handler( trans ).get_tag_by_id( trans, row[0] ) - # Exclude tags that are already applied to the item. + # Exclude tags that are already applied to the item. if ( item is not None ) and ( self.get_tag_handler( trans ).item_has_tag( trans, trans.user, item, tag ) ): continue # Add tag to autocomplete data. Use the most frequent name that user @@ -154,7 +154,7 @@ class TagsController ( BaseUIController, UsesTagsMixin ): item_class = item.__class__ item_tag_assoc_class = self.get_tag_handler( trans ).get_tag_assoc_class( item_class ) # Build select statement. - cols_to_select = [ item_tag_assoc_class.table.c.value, func.count( '*' ) ] + cols_to_select = [ item_tag_assoc_class.table.c.value, func.count( '*' ) ] from_obj = item_tag_assoc_class.table.join( item_class.table ).join( trans.app.model.Tag.table ) where_clause = and_( item_tag_assoc_class.table.c.user_id == user.id, trans.app.model.Tag.table.c.id == tag.id, diff --git a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py index c9bfa2d7b07..d4aa7e4e5db 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -137,7 +137,7 @@ class ToolRunner( BaseUIController ): #only allow rerunning if user is allowed access to the dataset. if not ( trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), data.dataset ) ): error( "You are not allowed to access this dataset" ) - # Get the associated job, if any. + # Get the associated job, if any. job = data.creating_job if not job: raise Exception("Failed to get job information for dataset hid %d" % data.hid) @@ -182,15 +182,15 @@ class ToolRunner( BaseUIController ): except: raise Exception( "Failed to get parameters for dataset id %d " % data.id ) upgrade_messages = tool.check_and_update_param_values( params_objects, trans, update_values=False ) - # Need to remap dataset parameters. Job parameters point to original - # dataset used; parameter should be the analygous dataset in the + # Need to remap dataset parameters. Job parameters point to original + # dataset used; parameter should be the analygous dataset in the # current history. history = trans.get_history() hda_source_dict = {} # Mapping from HDA in history to source HDAs. for hda in history.datasets: source_hda = hda.copied_from_history_dataset_association while source_hda:#should this check library datasets as well? - #FIXME: could be multiple copies of a hda in a single history, this does a better job of matching on cloned histories, + #FIXME: could be multiple copies of a hda in a single history, this does a better job of matching on cloned histories, #but is still less than perfect when eg individual datasets are copied between histories if source_hda not in hda_source_dict or source_hda.hid == hda.hid: hda_source_dict[ source_hda ] = hda @@ -246,7 +246,7 @@ class ToolRunner( BaseUIController ): add_frame=add_frame, tool_id_version_message=tool_id_version_message, **vars ) - + @web.expose def redirect( self, trans, redirect_url=None, **kwd ): if not redirect_url: diff --git a/lib/galaxy/webapps/galaxy/controllers/ucsc_proxy.py b/lib/galaxy/webapps/galaxy/controllers/ucsc_proxy.py index e7ae30f03b7..81edfd60db9 100644 --- a/lib/galaxy/webapps/galaxy/controllers/ucsc_proxy.py +++ b/lib/galaxy/webapps/galaxy/controllers/ucsc_proxy.py @@ -21,12 +21,12 @@ class UCSCProxy( BaseUIController ): if region not in [ 'genome', 'encode']: region = store.get('position','') if track == table: - display = 'UCSC: %s (%s)' % (track, region) + display = 'UCSC: %s (%s)' % (track, region) else: - display = 'UCSC: %s, %s (%s)' % (track, table, region) - return display - - @web.expose + display = 'UCSC: %s, %s (%s)' % (track, table, region) + return display + + @web.expose def index(self, trans, init=False, **kwd): base_url = None params = dict(kwd) @@ -40,7 +40,7 @@ class UCSCProxy( BaseUIController ): base_url = store.get(UCSC_URL, "http://genome.ucsc.edu/cgi-bin/hgTables?") params = dict(kwd) params['init'] = init - + if not init: for key, value in kwd.items(): store[key] = value @@ -48,7 +48,7 @@ class UCSCProxy( BaseUIController ): except: pass else: store = {} - + if init == "1": base_url = "http://genome.ucsc.edu/cgi-bin/hgTables?" params['db'] = 'hg17' @@ -59,11 +59,11 @@ class UCSCProxy( BaseUIController ): base_url = "http://archaea.ucsc.edu/cgi-bin/hgTables?" store[UCSC_URL] = base_url - + try: del params["__GALAXY__"] except: pass url = base_url + urllib.urlencode(params) - + page = urllib.urlopen(url) content = page.info().get('Content-type', '') except Exception, exc: @@ -81,13 +81,13 @@ class UCSCProxy( BaseUIController ): url = "/tool_runner/index?" + urllib.urlencode(params) trans.response.send_redirect(url) else: - try: + try: text = page.read() - + # Serialize store into a form element store_text = "" - + # Remove text regions that should not be exposed for key,value in altered_regions.items(): text = text.replace(key,value) @@ -114,7 +114,7 @@ class UCSCProxy( BaseUIController ): trans.log_event( "Proxy Error -> %s" % str(exc) ) msg = 'There has been a problem connecting to %s

    %s' % (base_url, exc) return msg - + # HTML for generating the proxy page. beginning = ''' @@ -146,7 +146,7 @@ ending = ''' # This is a mess of mappings of text to make the proxy friendlier to # galaxy users. -altered_regions = { +altered_regions = { '"../cgi-bin/hgTables' : '"/ucsc_proxy/index', '\noutput file:  (leave blank to keep output in browser)\n\nfile type returned:  plain text   gzip compressed' : '', '

    To reset all user cart settings (including custom tracks), \nclick here.' : '', @@ -154,6 +154,6 @@ altered_regions = { 'custom track' : 'custom track', '' : '', '' : '', - "

    %s
    ' % ( job.state, job.state ) def filter( self, trans, user, query, column_filter ): if column_filter == 'Unfinished': - return query.filter( not_( or_( model.Job.table.c.state == model.Job.states.OK, - model.Job.table.c.state == model.Job.states.ERROR, + return query.filter( not_( or_( model.Job.table.c.state == model.Job.states.OK, + model.Job.table.c.state == model.Job.states.ERROR, model.Job.table.c.state == model.Job.states.DELETED ) ) ) return query class ToolColumn( grids.TextColumn ): @@ -102,8 +102,8 @@ class SpecifiedDateListGrid( grids.Grid ): visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[1], columns[2] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[1], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -198,12 +198,12 @@ class Jobs( BaseUIController ): row.monitor_jobs, row.total_jobs, row.date.strftime( "%d" ) ) ) - return trans.fill_template( '/webapps/reports/jobs_specified_month_all.mako', - month_label=month_label, - year_label=year_label, - month=month, - jobs=jobs, - message=message ) + return trans.fill_template( '/webapps/reports/jobs_specified_month_all.mako', + month_label=month_label, + year_label=year_label, + month=month, + jobs=jobs, + message=message ) @web.expose def specified_month_in_error( self, trans, **kwd ): params = util.Params( kwd ) @@ -219,7 +219,7 @@ class Jobs( BaseUIController ): q = sa.select( ( sa.func.date( model.Job.table.c.create_time ).label( 'date' ), sa.func.count( model.Job.table.c.id ).label( 'total_jobs' ) ), whereclause = sa.and_( model.Job.table.c.state == 'error', - model.Job.table.c.create_time >= start_date, + model.Job.table.c.create_time >= start_date, model.Job.table.c.create_time < end_date ), from_obj = [ sa.outerjoin( model.Job.table, model.User.table ) ], group_by = [ 'date' ], @@ -230,11 +230,11 @@ class Jobs( BaseUIController ): row.date, row.total_jobs, row.date.strftime( "%d" ) ) ) - return trans.fill_template( '/webapps/reports/jobs_specified_month_in_error.mako', - month_label=month_label, - year_label=year_label, - month=month, - jobs=jobs, + return trans.fill_template( '/webapps/reports/jobs_specified_month_in_error.mako', + month_label=month_label, + year_label=year_label, + month=month, + jobs=jobs, message=message ) @web.expose def per_month_all( self, trans, **kwd ): @@ -251,7 +251,7 @@ class Jobs( BaseUIController ): for row in q.execute(): jobs.append( ( row.date.strftime( "%Y-%m" ), row.total_jobs - row.monitor_jobs, - row.monitor_jobs, + row.monitor_jobs, row.total_jobs, row.date.strftime( "%B" ), row.date.strftime( "%Y" ) ) ) @@ -270,7 +270,7 @@ class Jobs( BaseUIController ): order_by = [ sa.desc( 'date' ) ] ) jobs = [] for row in q.execute(): - jobs.append( ( row.date.strftime( "%Y-%m" ), + jobs.append( ( row.date.strftime( "%Y-%m" ), row.total_jobs, row.date.strftime( "%B" ), row.date.strftime( "%Y" ) ) ) @@ -288,7 +288,7 @@ class Jobs( BaseUIController ): group_by = [ 'user_email' ], order_by = [ sa.desc( 'total_jobs' ), 'user_email' ] ) for row in q.execute(): - jobs.append( ( row.user_email, + jobs.append( ( row.user_email, row.total_jobs ) ) return trans.fill_template( '/webapps/reports/jobs_per_user.mako', jobs=jobs, message=message ) @web.expose @@ -307,7 +307,7 @@ class Jobs( BaseUIController ): order_by = [ sa.desc( 'date' ) ] ) jobs = [] for row in q.execute(): - jobs.append( ( row.date.strftime( "%Y-%m" ), + jobs.append( ( row.date.strftime( "%Y-%m" ), row.total_jobs, row.date.strftime( "%B" ), row.date.strftime( "%Y" ) ) ) @@ -325,7 +325,7 @@ class Jobs( BaseUIController ): group_by = [ 'tool_id' ], order_by = [ 'tool_id' ] ) for row in q.execute(): - jobs.append( ( row.tool_id, + jobs.append( ( row.tool_id, row.total_jobs ) ) return trans.fill_template( '/webapps/reports/jobs_per_tool.mako', jobs=jobs, @@ -338,13 +338,13 @@ class Jobs( BaseUIController ): specified_date = params.get( 'specified_date', datetime.utcnow().strftime( "%Y-%m-%d" ) ) q = sa.select( ( sa.func.date_trunc( 'month', sa.func.date( model.Job.table.c.create_time ) ).label( 'date' ), sa.func.count( model.Job.table.c.id ).label( 'total_jobs' ) ), - whereclause = model.Job.table.c.tool_id == tool_id, + whereclause = model.Job.table.c.tool_id == tool_id, from_obj = [ model.Job.table ], group_by = [ sa.func.date_trunc( 'month', sa.func.date( model.Job.table.c.create_time ) ) ], order_by = [ sa.desc( 'date' ) ] ) jobs = [] for row in q.execute(): - jobs.append( ( row.date.strftime( "%Y-%m" ), + jobs.append( ( row.date.strftime( "%Y-%m" ), row.total_jobs, row.date.strftime( "%B" ), row.date.strftime( "%Y" ) ) ) diff --git a/lib/galaxy/webapps/reports/controllers/sample_tracking.py b/lib/galaxy/webapps/reports/controllers/sample_tracking.py index 02e41ae7ad9..bf63cb5dba5 100644 --- a/lib/galaxy/webapps/reports/controllers/sample_tracking.py +++ b/lib/galaxy/webapps/reports/controllers/sample_tracking.py @@ -79,8 +79,8 @@ class SpecifiedDateListGrid( grids.Grid ): model_class=model.User, visible=False ), ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[2] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -157,7 +157,7 @@ class SampleTracking( BaseUIController ): group_by = [ 'user_email' ], order_by = [ sa.desc( 'total' ), 'user_email' ] ) for row in q.execute(): - requests.append( ( row.user_email, + requests.append( ( row.user_email, row.total ) ) return trans.fill_template( '/webapps/reports/requests_per_user.mako', requests=requests, message=message ) @web.expose @@ -174,7 +174,7 @@ class SampleTracking( BaseUIController ): order_by = [ sa.desc( 'date' ) ] ) requests = [] for row in q.execute(): - requests.append( ( row.date.strftime( "%Y-%m" ), + requests.append( ( row.date.strftime( "%Y-%m" ), row.total, row.date.strftime( "%B" ), row.date.strftime( "%Y" ) ) ) diff --git a/lib/galaxy/webapps/reports/controllers/users.py b/lib/galaxy/webapps/reports/controllers/users.py index 3dd10da30bf..543dc891fc1 100644 --- a/lib/galaxy/webapps/reports/controllers/users.py +++ b/lib/galaxy/webapps/reports/controllers/users.py @@ -27,7 +27,7 @@ class Users( BaseUIController ): order_by = [ sa.desc( 'date' ) ] ) users = [] for row in q.execute(): - users.append( ( row.date.strftime( "%Y-%m" ), + users.append( ( row.date.strftime( "%Y-%m" ), row.num_users, row.date.strftime( "%B" ), row.date.strftime( "%Y" ) ) ) @@ -55,14 +55,14 @@ class Users( BaseUIController ): users = [] for row in q.execute(): users.append( ( row.date.strftime( "%Y-%m-%d" ), - row.date.strftime( "%d" ), - row.num_users, + row.date.strftime( "%d" ), + row.num_users, row.date.strftime( "%A" ) ) ) - return trans.fill_template( '/webapps/reports/registered_users_specified_month.mako', - month_label=month_label, - year_label=year_label, - month=month, - users=users, + return trans.fill_template( '/webapps/reports/registered_users_specified_month.mako', + month_label=month_label, + year_label=year_label, + month=month, + users=users, message=message ) @web.expose def specified_date( self, trans, **kwd ): @@ -85,13 +85,13 @@ class Users( BaseUIController ): users = [] for row in q.execute(): users.append( ( row.email ) ) - return trans.fill_template( '/webapps/reports/registered_users_specified_date.mako', - specified_date=start_date, - day_label=day_label, - month_label=month_label, - year_label=year_label, - day_of_month=day_of_month, - users=users, + return trans.fill_template( '/webapps/reports/registered_users_specified_date.mako', + specified_date=start_date, + day_label=day_label, + month_label=month_label, + year_label=year_label, + day_of_month=day_of_month, + users=users, message=message ) @web.expose def last_access_date( self, trans, **kwd ): diff --git a/lib/galaxy/webapps/reports/controllers/workflows.py b/lib/galaxy/webapps/reports/controllers/workflows.py index b506b6ed1d8..0b318e05d81 100644 --- a/lib/galaxy/webapps/reports/controllers/workflows.py +++ b/lib/galaxy/webapps/reports/controllers/workflows.py @@ -79,8 +79,8 @@ class SpecifiedDateListGrid( grids.Grid ): model_class=model.User, visible=False ), ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[2] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -157,7 +157,7 @@ class Workflows( BaseUIController ): group_by = [ 'user_email' ], order_by = [ sa.desc( 'total_workflows' ), 'user_email' ] ) for row in q.execute(): - workflows.append( ( row.user_email, + workflows.append( ( row.user_email, row.total_workflows ) ) return trans.fill_template( '/webapps/reports/workflows_per_user.mako', workflows=workflows, message=message ) @web.expose @@ -174,7 +174,7 @@ class Workflows( BaseUIController ): order_by = [ sa.desc( 'date' ) ] ) workflows = [] for row in q.execute(): - workflows.append( ( row.date.strftime( "%Y-%m" ), + workflows.append( ( row.date.strftime( "%Y-%m" ), row.total_workflows, row.date.strftime( "%B" ), row.date.strftime( "%Y" ) ) ) diff --git a/lib/galaxy/webapps/tool_shed/api/repositories.py b/lib/galaxy/webapps/tool_shed/api/repositories.py index acdb2d25859..623b8fc64a7 100644 --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -34,10 +34,10 @@ class RepositoriesController( BaseAPIController ): def get_ordered_installable_revisions( self, trans, name, owner, **kwd ): """ GET /api/repositories/get_ordered_installable_revisions - + :param name: the name of the Repository :param owner: the owner of the Repository - + Returns the ordered list of changeset revision hash strings that are associated with installable revisions. As in the changelog, the list is ordered oldest to newest. """ @@ -60,50 +60,50 @@ class RepositoriesController( BaseAPIController ): def get_repository_revision_install_info( self, trans, name, owner, changeset_revision, **kwd ): """ GET /api/repositories/get_repository_revision_install_info - + :param name: the name of the Repository :param owner: the owner of the Repository :param changset_revision: the changset_revision of the RepositoryMetadata object associated with the Repository - + Returns a list of the following dictionaries:: - a dictionary defining the Repository. For example: { - "deleted": false, - "deprecated": false, - "description": "add_column hello", - "id": "f9cad7b01a472135", - "long_description": "add_column hello", - "name": "add_column", - "owner": "test", - "private": false, - "times_downloaded": 6, - "url": "/api/repositories/f9cad7b01a472135", + "deleted": false, + "deprecated": false, + "description": "add_column hello", + "id": "f9cad7b01a472135", + "long_description": "add_column hello", + "name": "add_column", + "owner": "test", + "private": false, + "times_downloaded": 6, + "url": "/api/repositories/f9cad7b01a472135", "user_id": "f9cad7b01a472135" } - a dictionary defining the Repository revision (RepositoryMetadata). For example: { - "changeset_revision": "3a08cc21466f", - "downloadable": true, - "has_repository_dependencies": false, - "id": "f9cad7b01a472135", - "includes_datatypes": false, - "includes_tool_dependencies": false, + "changeset_revision": "3a08cc21466f", + "downloadable": true, + "has_repository_dependencies": false, + "id": "f9cad7b01a472135", + "includes_datatypes": false, + "includes_tool_dependencies": false, "includes_tools": true, "includes_tools_for_display_in_tool_panel": true, - "includes_workflows": false, - "malicious": false, - "repository_id": "f9cad7b01a472135", + "includes_workflows": false, + "malicious": false, + "repository_id": "f9cad7b01a472135", "url": "/api/repository_revisions/f9cad7b01a472135" } - a dictionary including the additional information required to install the repository. For example: { "add_column": [ - "add_column hello", - "http://test@localhost:9009/repos/test/add_column", - "3a08cc21466f", - "1", - "test", - {}, + "add_column hello", + "http://test@localhost:9009/repos/test/add_column", + "3a08cc21466f", + "1", + "test", + {}, {} ] } @@ -181,7 +181,7 @@ class RepositoriesController( BaseAPIController ): """ GET /api/repositories/{encoded_repository_id} Returns information about a repository in the Tool Shed. - + :param id: the encoded id of the Repository object """ # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 diff --git a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py index 8aa0f0ebd31..09c61de4a25 100644 --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -27,7 +27,7 @@ class RepositoryRevisionsController( BaseAPIController ): """ POST /api/repository_revisions/export Creates and saves a gzip compressed tar archive of a repository and optionally all of it's repository dependencies. - + The following parameters are included in the payload. :param tool_shed_url (required): the base URL of the Tool Shed from which the Repository was installed :param name (required): the name of the Repository @@ -145,7 +145,7 @@ class RepositoryRevisionsController( BaseAPIController ): """ GET /api/repository_revisions/{encoded_repository_metadata_id} Displays information about a repository_metadata record in the Tool Shed. - + :param id: the encoded id of the `RepositoryMetadata` object """ # Example URL: http://localhost:9009/api/repository_revisions/bb125606ff9ea620 diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index cf1ec600619..6b1027839a0 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -29,7 +29,7 @@ class CommunityWebApplication( galaxy.web.framework.WebApplication ): def add_ui_controllers( webapp, app ): """ - Search for controllers in the 'galaxy.webapps.controllers' module and add + Search for controllers in the 'galaxy.webapps.controllers' module and add them to the webapp. """ from galaxy.web.base.controller import BaseUIController @@ -125,7 +125,7 @@ def wrap_in_middleware( app, global_conf, **local_conf ): display_servers = util.listify( conf.get( 'display_servers', '' ) ), admin_users = conf.get( 'admin_users', '' ).split( ',' ) ) log.debug( "Enabling 'remote user' middleware" ) - # The recursive middleware allows for including requests in other + # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): from paste import recursive @@ -177,7 +177,7 @@ def wrap_in_middleware( app, global_conf, **local_conf ): app = hg.Hg( app, conf ) log.debug( "Enabling hg middleware" ) return app - + def wrap_in_static( app, global_conf, **local_conf ): from paste.urlmap import URLMap from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static @@ -199,7 +199,7 @@ def wrap_in_static( app, global_conf, **local_conf ): urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir" ), cache_time ) # URL mapper becomes the root webapp return urlmap - + def build_template_error_formatters(): """ Build a list of template error formatters for WebError. When an error diff --git a/lib/galaxy/webapps/tool_shed/config.py b/lib/galaxy/webapps/tool_shed/config.py index 936ae51eb91..c51de96070d 100644 --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -33,7 +33,7 @@ class Configuration( object ): # Database related configuration self.database = resolve_path( kwargs.get( "database_file", "database/community.sqlite" ), self.root ) self.database_connection = kwargs.get( "database_connection", False ) - self.database_engine_options = get_database_engine_options( kwargs ) + self.database_engine_options = get_database_engine_options( kwargs ) self.database_create_tables = string_as_bool( kwargs.get( "database_create_tables", "True" ) ) # Where dataset files are stored self.file_path = resolve_path( kwargs.get( "file_path", "database/community_files" ), self.root ) @@ -200,7 +200,7 @@ def configure_logging( config ): if level <= logging.DEBUG: logging.getLogger( "paste.httpserver.ThreadPool" ).setLevel( logging.WARN ) # Remove old handlers - for h in root.handlers[:]: + for h in root.handlers[:]: root.removeHandler(h) # Create handler if destination == "stdout": @@ -208,7 +208,7 @@ def configure_logging( config ): else: handler = logging.FileHandler( destination ) # Create formatter - formatter = logging.Formatter( format ) + formatter = logging.Formatter( format ) # Hook everything up handler.setFormatter( formatter ) root.addHandler( handler ) diff --git a/lib/galaxy/webapps/tool_shed/controllers/admin.py b/lib/galaxy/webapps/tool_shed/controllers/admin.py index 11e16619d64..eaa1542b32d 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/admin.py +++ b/lib/galaxy/webapps/tool_shed/controllers/admin.py @@ -16,7 +16,7 @@ log = logging.getLogger( __name__ ) class AdminController( BaseUIController, Admin ): - + user_list_grid = admin_grids.UserGrid() role_list_grid = admin_grids.RoleGrid() group_list_grid = admin_grids.GroupGrid() @@ -77,7 +77,7 @@ class AdminController( BaseUIController, Admin ): return self.undelete_repository( trans, **kwd ) # The changeset_revision_select_field in the RepositoryGrid performs a refresh_on_change # which sends in request parameters like changeset_revison_1, changeset_revision_2, etc. One - # of the many select fields on the grid performed the refresh_on_change, so we loop through + # of the many select fields on the grid performed the refresh_on_change, so we loop through # all of the received values to see which value is not the repository tip. If we find it, we # know the refresh_on_change occurred, and we have the necessary repository id and change set # revision to pass on. @@ -154,7 +154,7 @@ class AdminController( BaseUIController, Admin ): status = kwd.get( 'status', 'done' ) id = kwd.get( 'id', None ) if id: - # Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. + # Deleting multiple items is currently not allowed (allow_multiple=False), so there will only be 1 id. ids = util.listify( id ) count = 0 deleted_repositories = "" diff --git a/lib/galaxy/webapps/tool_shed/controllers/hg.py b/lib/galaxy/webapps/tool_shed/controllers/hg.py index 506604fbe44..bf0d2d0d39f 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -24,9 +24,9 @@ class HgController( BaseUIController ): hgwebapp = hgwebdir( hgweb_config ) return hgwebapp wsgi_app = wsgiapplication( make_web_app ) - if hg_version >= '2.2.3' and cmd == 'pushkey': + if hg_version >= '2.2.3' and cmd == 'pushkey': # When doing an "hg push" from the command line, the following commands, in order, will be retrieved from environ, depending - # upon the mercurial version being used. In mercurial version 2.2.3, section 15.2. Command changes includes a new feature: + # upon the mercurial version being used. In mercurial version 2.2.3, section 15.2. Command changes includes a new feature: # pushkey: add hooks for pushkey/listkeys (see http://mercurial.selenic.com/wiki/WhatsNew#Mercurial_2.2.3_.282012-07-01.29). # We require version 2.2.3 since the pushkey hook was added in that version. # If mercurial version >= '2.2.3': capabilities -> batch -> branchmap -> unbundle -> listkeys -> pushkey diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 4ae5b99817f..a9f5681c1d3 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -832,7 +832,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): update = 'true' no_update = 'false' elif galaxy_url: - # Start building up the url to redirect back to the calling Galaxy instance. + # Start building up the url to redirect back to the calling Galaxy instance. url = suc.url_join( galaxy_url, 'admin_toolshed/update_to_changeset_revision?tool_shed_url=%s&name=%s&owner=%s&changeset_revision=%s&latest_changeset_revision=' % \ ( web.url_for( '/', qualified=True ), repository.name, repository.user.username, changeset_revision ) ) @@ -846,7 +846,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): # Return the same value for changeset_revision and latest_changeset_revision. url += latest_changeset_revision else: - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), changeset_revision ) if repository_metadata: @@ -993,7 +993,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): @web.require_login( "deprecate repository" ) def deprecate( self, trans, **kwd ): """Mark a repository in the tool shed as deprecated or not deprecated.""" - # Marking a repository in the tool shed as deprecated has no effect on any downloadable changeset revisions that may be associated with the + # Marking a repository in the tool shed as deprecated has no effect on any downloadable changeset revisions that may be associated with the # repository. Revisions are not marked as not downlaodable because those that have installed the repository must be allowed to get updates. message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) @@ -1006,7 +1006,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): if mark_deprecated: message = 'The repository %s has been marked as deprecated.' % repository.name else: - message = 'The repository %s has been marked as not deprecated.' % repository.name + message = 'The repository %s has been marked as not deprecated.' % repository.name trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', operation='repositories_i_own', @@ -1373,7 +1373,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): owner = kwd.get( 'owner', None ) changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), changeset_revision ) includes_data_managers, includes_datatypes, includes_tools, includes_tools_for_display_in_tool_panel, includes_tool_dependencies, has_repository_dependencies, includes_workflows = \ @@ -1536,15 +1536,15 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): # The guid attribute in an RSS feed's list of items allows a feed reader to choose not to show an item as updated # if the guid is unchanged. For functional test results, the citable URL is sufficiently unique to enable # that behavior. - functional_test_results.append( dict( title=title, - guid=repository_citable_url, - link=repository_citable_url, - description='\n'.join( description_lines ), + functional_test_results.append( dict( title=title, + guid=repository_citable_url, + link=repository_citable_url, + description='\n'.join( description_lines ), pubdate=time_tested ) ) trans.response.set_content_type( 'application/rss+xml' ) - return trans.fill_template( '/rss.mako', - title='Tool functional test results', - link=tool_shed_url, + return trans.fill_template( '/rss.mako', + title='Tool functional test results', + link=tool_shed_url, description='Functional test results for repositories owned by %s.' % user.username, pubdate=strftime( '%a, %d %b %Y %H:%M:%S UT', gmtime() ), items=functional_test_results ) @@ -1558,7 +1558,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): @web.json def get_readme_files( self, trans, **kwd ): """ - This method is called when installing or re-installing a single repository into a Galaxy instance. If the received changeset_revision + This method is called when installing or re-installing a single repository into a Galaxy instance. If the received changeset_revision includes one or more readme files, return them in a dictionary. """ repository_name = kwd[ 'name' ] @@ -1612,7 +1612,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): includes_tool_dependencies = False repo_info_dicts = [] for tup in zip( util.listify( repository_ids ), util.listify( changeset_revisions ) ): - repository_id, changeset_revision = tup + repository_id, changeset_revision = tup repo_info_dict, cur_includes_tools, cur_includes_tool_dependencies, cur_includes_tools_for_display_in_tool_panel, cur_has_repository_dependencies = \ repository_util.get_repo_info_dict( trans, repository_id, changeset_revision ) if cur_has_repository_dependencies and not has_repository_dependencies: @@ -2354,7 +2354,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): rra = self.get_user_item_rating( trans.sa_session, trans.user, repository, webapp_model=trans.model ) metadata = self.get_metadata( trans, id, repository.tip( trans.app ) ) repository_type_select_field = rt_util.build_repository_type_select_field( trans, repository=repository ) - return trans.fill_template( '/webapps/tool_shed/repository/rate_repository.mako', + return trans.fill_template( '/webapps/tool_shed/repository/rate_repository.mako', repository=repository, metadata=metadata, avg_rating=avg_rating, @@ -2445,7 +2445,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): if tip == repository.tip( trans.app ): message += 'No changes to repository. ' kwd[ 'message' ] = message - + else: message += 'The selected files were deleted from the repository. ' kwd[ 'message' ] = message @@ -2707,7 +2707,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): # Make sure we'll view latest changeset first. changesets.insert( 0, change_dict ) metadata = self.get_metadata( trans, id, repository.tip( trans.app ) ) - return trans.fill_template( '/webapps/tool_shed/repository/view_changelog.mako', + return trans.fill_template( '/webapps/tool_shed/repository/view_changelog.mako', repository=repository, metadata=metadata, changesets=changesets, @@ -2754,7 +2754,7 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ): next = "%s:%s" % ( ctx_child_rev, ctx_child ) else: next = None - return trans.fill_template( '/webapps/tool_shed/repository/view_changeset.mako', + return trans.fill_template( '/webapps/tool_shed/repository/view_changeset.mako', repository=repository, metadata=metadata, prev=prev, diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py index 6a8ebaa7313..9015296c107 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -18,9 +18,9 @@ from mercurial import hg, ui, patch, commands log = logging.getLogger( __name__ ) - + class RepositoryReviewController( BaseUIController, ratings_util.ItemRatings ): - + component_grid = repository_review_grids.ComponentGrid() repositories_ready_for_review_grid = repository_review_grids.RepositoriesReadyForReviewGrid() repositories_reviewed_by_me_grid = repository_review_grids.RepositoriesReviewedByMeGrid() diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py index e5c988f0dfc..51919e19f4c 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -100,7 +100,7 @@ class UploadController( BaseUIController ): if isempty: tar = None istar = False - else: + else: # Determine what we have - a single file or an archive try: if ( isgzip or isbz2 ) and uncompress_file: diff --git a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py index 91a957b44fe..9150d6e8991 100644 --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py @@ -42,7 +42,7 @@ class Hg( object ): cmd = self.__get_hg_command( **environ ) if cmd == 'changegroup': # This is an hg clone from the command line. When doing this, the following 5 commands, in order, - # will be retrieved from environ: + # will be retrieved from environ: # between -> heads -> changegroup -> capabilities -> listkeys # # Increment the value of the times_downloaded column in the repository table for the cloned repository. @@ -69,22 +69,22 @@ class Hg( object ): # The mercurial API unbundle() ( i.e., hg push ) and pushkey() methods ultimately require authorization. # We'll force password entry every time a change set is pushed. # - # When a user executes hg commit, it is not guaranteed to succeed. Mercurial records your name - # and address with each change that you commit, so that you and others will later be able to - # tell who made each change. Mercurial tries to automatically figure out a sensible username + # When a user executes hg commit, it is not guaranteed to succeed. Mercurial records your name + # and address with each change that you commit, so that you and others will later be able to + # tell who made each change. Mercurial tries to automatically figure out a sensible username # to commit the change with. It will attempt each of the following methods, in order: # - # 1) If you specify a -u option to the hg commit command on the command line, followed by a username, + # 1) If you specify a -u option to the hg commit command on the command line, followed by a username, # this is always given the highest precedence. # 2) If you have set the HGUSER environment variable, this is checked next. - # 3) If you create a file in your home directory called .hgrc with a username entry, that + # 3) If you create a file in your home directory called .hgrc with a username entry, that # will be used next. # 4) If you have set the EMAIL environment variable, this will be used next. - # 5) Mercurial will query your system to find out your local user name and host name, and construct - # a username from these components. Since this often results in a username that is not very useful, + # 5) Mercurial will query your system to find out your local user name and host name, and construct + # a username from these components. Since this often results in a username that is not very useful, # it will print a warning if it has to do this. # - # If all of these mechanisms fail, Mercurial will fail, printing an error message. In this case, it + # If all of these mechanisms fail, Mercurial will fail, printing an error message. In this case, it # will not let you commit until you set up a username. result = self.authentication( environ ) if not isinstance( result, str ) and cmd == 'unbundle' and 'wsgi.input' in environ: diff --git a/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py b/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py index 3b0342054e1..cfca35e853b 100644 --- a/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py +++ b/lib/galaxy/webapps/tool_shed/framework/middleware/remoteuser.py @@ -68,7 +68,7 @@ class RemoteUser( object ): an email address) was provided by the upstream (proxy) server. Since tool shed usernames are email addresses, a default mail domain must be set.

    The variable remote_user_maildomain must be set before you - can access this tool shed. Contact your local tool shed administrator. + can access this tool shed. Contact your local tool shed administrator. """ return self.error( start_response, title, message ) return self.app( environ, start_response ) diff --git a/lib/galaxy/webapps/tool_shed/model/__init__.py b/lib/galaxy/webapps/tool_shed/model/__init__.py index de75f7ea51c..2a5eaccdeeb 100644 --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -74,7 +74,7 @@ class Role( object, DictifiableMixin ): dict_collection_visible_keys = ( 'id', 'name' ) dict_element_visible_keys = ( 'id', 'name', 'description', 'type' ) private_id = None - types = Bunch( + types = Bunch( PRIVATE = 'private', SYSTEM = 'system', USER = 'user', @@ -109,15 +109,15 @@ class GroupRoleAssociation( object ): class GalaxySession( object ): - def __init__( self, - id=None, - user=None, - remote_host=None, - remote_addr=None, - referer=None, - current_history=None, - session_key=None, - is_valid=False, + def __init__( self, + id=None, + user=None, + remote_host=None, + remote_addr=None, + referer=None, + current_history=None, + session_key=None, + is_valid=False, prev_session_id=None ): self.id = id self.user = user @@ -251,7 +251,7 @@ class RepositoryMetadata( object, DictifiableMixin ): 'test_install_error', 'time_last_tested', 'tool_test_results', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) - def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, + def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, missing_test_components=None, tools_functionally_correct=False, do_not_test=False, test_install_error=False, time_last_tested=None, tool_test_results=None, has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ): diff --git a/lib/galaxy/webapps/tool_shed/model/mapping.py b/lib/galaxy/webapps/tool_shed/model/mapping.py index f8f0669ea90..902d6e296c7 100644 --- a/lib/galaxy/webapps/tool_shed/model/mapping.py +++ b/lib/galaxy/webapps/tool_shed/model/mapping.py @@ -1,6 +1,6 @@ """ Details of how the data model objects are mapped onto the relational database -are encapsulated here. +are encapsulated here. """ import logging log = logging.getLogger( __name__ ) @@ -24,7 +24,7 @@ context = Session = scoped_session( sessionmaker( autoflush=False, autocommit=Tr context.current = Session # NOTE REGARDING TIMESTAMPS: -# It is currently difficult to have the timestamps calculated by the +# It is currently difficult to have the timestamps calculated by the # database in a portable way, so we're doing it in the client. This # also saves us from needing to postfetch on postgres. HOWEVER: it # relies on the client's clock being set correctly, so if clustering @@ -67,7 +67,7 @@ Role.table = Table( "role", metadata, Column( "type", String( 40 ), index=True ), Column( "deleted", Boolean, index=True, default=False ) ) -UserGroupAssociation.table = Table( "user_group_association", metadata, +UserGroupAssociation.table = Table( "user_group_association", metadata, Column( "id", Integer, primary_key=True ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "group_id", Integer, ForeignKey( "galaxy_group.id" ), index=True ), @@ -200,16 +200,16 @@ Tag.table = Table( "tag", metadata, Column( "id", Integer, primary_key=True ), Column( "type", Integer ), Column( "parent_id", Integer, ForeignKey( "tag.id" ) ), - Column( "name", TrimmedString(255) ), + Column( "name", TrimmedString(255) ), UniqueConstraint( "name" ) ) # With the tables defined we can define the mappers and setup the relationships between the model objects. -mapper( User, User.table, +mapper( User, User.table, properties=dict( active_repositories=relation( Repository, primaryjoin=( ( Repository.table.c.user_id == User.table.c.id ) & ( not_( Repository.table.c.deleted ) ) ), order_by=( Repository.table.c.name ) ), galaxy_sessions=relation( GalaxySession, order_by=desc( GalaxySession.table.c.update_time ) ), api_keys=relation( APIKeys, backref="user", order_by=desc( APIKeys.table.c.create_time ) ) ) ) -mapper( APIKeys, APIKeys.table, +mapper( APIKeys, APIKeys.table, properties = {} ) mapper( Group, Group.table, @@ -227,7 +227,7 @@ mapper( UserGroupAssociation, UserGroupAssociation.table, mapper( UserRoleAssociation, UserRoleAssociation.table, properties=dict( user=relation( User, backref="roles" ), - non_private_roles=relation( User, + non_private_roles=relation( User, backref="non_private_roles", primaryjoin=( ( User.table.c.id == UserRoleAssociation.table.c.user_id ) & ( UserRoleAssociation.table.c.role_id == Role.table.c.id ) & not_( Role.table.c.name == User.table.c.email ) ) ), role=relation( Role ) ) ) @@ -249,7 +249,7 @@ mapper( Category, Category.table, primaryjoin=( Category.table.c.id == RepositoryCategoryAssociation.table.c.category_id ), secondaryjoin=( RepositoryCategoryAssociation.table.c.repository_id == Repository.table.c.id ) ) ) ) -mapper( Repository, Repository.table, +mapper( Repository, Repository.table, properties = dict( categories=relation( RepositoryCategoryAssociation ), ratings=relation( RepositoryRatingAssociation, order_by=desc( RepositoryRatingAssociation.table.c.update_time ), backref="repositories" ), diff --git a/lib/galaxy/webapps/tool_shed/model/migrate/check.py b/lib/galaxy/webapps/tool_shed/model/migrate/check.py index be1e61d6adf..3a67a54c56e 100644 --- a/lib/galaxy/webapps/tool_shed/model/migrate/check.py +++ b/lib/galaxy/webapps/tool_shed/model/migrate/check.py @@ -26,12 +26,12 @@ def create_or_verify_database( url, engine_options={} ): Check that the database is use-able, possibly creating it if empty (this is the only time we automatically create tables, otherwise we force the user to do it using the management script so they can create backups). - + 1) Empty database --> initialize with latest version and return 2) Database older than migration support --> fail and require manual update 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update) 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade" - + """ dialect = ( url.split( ':', 1 ) )[0] try: @@ -84,7 +84,7 @@ def create_or_verify_database( url, engine_options={} ): raise Exception( exception_msg ) else: log.info( "At database version %d" % db_schema.version ) - + def migrate_to_current_version( engine, schema ): # Changes to get to current version changeset = schema.changeset( None ) diff --git a/lib/galaxy/webapps/tool_shed/security/__init__.py b/lib/galaxy/webapps/tool_shed/security/__init__.py index 0ac08a0674d..8bb2b2a9987 100644 --- a/lib/galaxy/webapps/tool_shed/security/__init__.py +++ b/lib/galaxy/webapps/tool_shed/security/__init__.py @@ -112,7 +112,7 @@ class CommunityRBACAgent( RBACAgent ): return [ permission for permission in item.actions if permission.action == action.action ] def get_private_user_role( self, user, auto_create=False ): role = self.sa_session.query( self.model.Role ) \ - .filter( and_( self.model.Role.table.c.name == user.email, + .filter( and_( self.model.Role.table.c.name == user.email, self.model.Role.table.c.type == self.model.Role.types.PRIVATE ) ) \ .first() if not role: @@ -123,7 +123,7 @@ class CommunityRBACAgent( RBACAgent ): return role def get_repository_reviewer_role( self ): return self.sa_session.query( self.model.Role ) \ - .filter( and_( self.model.Role.table.c.name == 'Repository Reviewer', + .filter( and_( self.model.Role.table.c.name == 'Repository Reviewer', self.model.Role.table.c.type == self.model.Role.types.SYSTEM ) ) \ .first() def set_entity_group_associations( self, groups=[], users=[], roles=[], delete_existing_assocs=True ): diff --git a/lib/galaxy/webapps/tool_shed/util/hgweb_config.py b/lib/galaxy/webapps/tool_shed/util/hgweb_config.py index 0730eb879b6..57281e9b2a3 100644 --- a/lib/galaxy/webapps/tool_shed/util/hgweb_config.py +++ b/lib/galaxy/webapps/tool_shed/util/hgweb_config.py @@ -90,4 +90,3 @@ class HgWebConfigManager( object ): config_file = open( self.hgweb_config, 'wb' ) self.in_memory_config.write( config_file ) config_file.close - \ No newline at end of file diff --git a/lib/galaxy_utils/sequence/fasta.py b/lib/galaxy_utils/sequence/fasta.py index 4917536c15b..b40a3157f61 100644 --- a/lib/galaxy_utils/sequence/fasta.py +++ b/lib/galaxy_utils/sequence/fasta.py @@ -20,7 +20,7 @@ class fastaReader( object ): while line and line.startswith( '#' ): line = self.file.readline() if not line: - raise StopIteration + raise StopIteration assert line.startswith( '>' ), "FASTA headers must start with >" rval = fastaSequence() rval.identifier = line.strip() @@ -31,7 +31,7 @@ class fastaReader( object ): if line: self.file.seek( offset ) #this causes sequence id lines to be read twice, once to determine previous sequence end and again when getting actual sequence; can we cache this to prevent it from being re-read? return rval - #454 qual test data that was used has decimal scores that don't have trailing spaces + #454 qual test data that was used has decimal scores that don't have trailing spaces #so we'll need to parse and build these sequences not based upon de facto standards #i.e. in a less than ideal fashion line = line.rstrip() diff --git a/lib/galaxy_utils/sequence/fastq.py b/lib/galaxy_utils/sequence/fastq.py index e9b314e75c5..4a9962cbc68 100644 --- a/lib/galaxy_utils/sequence/fastq.py +++ b/lib/galaxy_utils/sequence/fastq.py @@ -182,7 +182,7 @@ class fastqSolexaRead( fastqSequencingRead ): quality_max = 62 score_system = 'solexa' sequence_space = 'base' - + class fastqCSSangerRead( fastqSequencingRead ): format = 'cssanger' #color space ascii_min = 33 @@ -227,7 +227,7 @@ class fastqCSSangerRead( fastqSequencingRead ): rval.sequence = self.color_space_converter.to_color_space( transform.reverse( self.color_space_converter.to_base_space( rval.sequence ) ), adapter_base = adapter ) else: rval.sequence = transform.reverse( rval.sequence ) - + if rval.is_ascii_encoded(): rval.quality = rval.quality[::-1] else: @@ -264,7 +264,7 @@ class fastqCSSangerRead( fastqSequencingRead ): elif new_adapter: rval.sequence = "%s%s" % ( new_adapter, rval.sequence ) return rval - def apply_galaxy_conventions( self ): + def apply_galaxy_conventions( self ): if self.has_adapter_base() and len( self.sequence ) == len( self.get_ascii_quality_scores() ): #SRA adds FAKE/DUMMY quality scores to the adapter base, we remove them here if self.is_ascii_encoded(): self.quality = self.quality[1:] @@ -366,7 +366,7 @@ class fastqAggregator( object ): return [ halfed ] return[ halfed - 1, halfed ] read_count = self.get_read_count_for_column( i ) - + min_score = self.get_score_min_for_column( i ) max_score = self.get_score_max_for_column( i ) sum_score = self.get_score_sum_for_column( i ) @@ -388,7 +388,7 @@ class fastqAggregator( object ): #determine iqr and step iqr = q3 - q1 step = 1.5 * iqr - + #Determine whiskers and outliers outliers = [] score_list = sorted( self.get_score_list_for_column( i ) ) @@ -399,7 +399,7 @@ class fastqAggregator( object ): break else: outliers.append( score ) - + right_whisker = q3 + step score_list.reverse() for score in score_list: @@ -408,17 +408,17 @@ class fastqAggregator( object ): break else: outliers.append( score ) - - column_stats = { 'read_count': read_count, - 'min_score': min_score, - 'max_score': max_score, - 'sum_score': sum_score, - 'mean_score': mean_score, - 'q1': q1, - 'med_score': med_score, - 'q3': q3, - 'iqr': iqr, - 'left_whisker': left_whisker, + + column_stats = { 'read_count': read_count, + 'min_score': min_score, + 'max_score': max_score, + 'sum_score': sum_score, + 'mean_score': mean_score, + 'q1': q1, + 'med_score': med_score, + 'q3': q3, + 'iqr': iqr, + 'left_whisker': left_whisker, 'right_whisker': right_whisker, 'outliers': outliers } return column_stats @@ -439,9 +439,9 @@ class fastqReader( object ): #remove empty lines, apparently extra new lines at end of file is common? if fastq_header: break - + assert fastq_header.startswith( '@' ), 'Invalid fastq header: %s' % fastq_header - rval = fastqSequencingRead.get_class_by_format( self.format )() + rval = fastqSequencingRead.get_class_by_format( self.format )() rval.identifier = fastq_header while True: line = self.file.readline() @@ -483,7 +483,7 @@ class fastqVerboseErrorReader( fastqReader ): def next( self ): last_good_end_offset = self.file.tell() last_readline_count = self.file.readline_count - try: + try: block = super( fastqVerboseErrorReader, self ).next() self.last_good_identifier = block.identifier return block @@ -601,7 +601,7 @@ class fastqJoiner( object ): identifier = read1_id if read1_desc: identifier = identifier + ' ' + read1_desc - + #use force quality encoding, if not present force to encoding of first read force_quality_encoding = self.force_quality_encoding if not force_quality_encoding: @@ -609,7 +609,7 @@ class fastqJoiner( object ): force_quality_encoding = 'ascii' else: force_quality_encoding = 'decimal' - + new_read1 = read1.convert_read_to_format( self.format, force_quality_encoding = force_quality_encoding ) new_read2 = read2.convert_read_to_format( self.format, force_quality_encoding = force_quality_encoding ) rval = FASTQ_FORMATS[ self.format ]() @@ -649,7 +649,7 @@ class fastqJoiner( object ): is_first = False return is_first -class fastqSplitter( object ): +class fastqSplitter( object ): def split( self, fastq_read ): length = len( fastq_read ) #Only reads of even lengths can be split diff --git a/lib/galaxy_utils/sequence/vcf.py b/lib/galaxy_utils/sequence/vcf.py index 491f3677276..4bbd9f5d788 100644 --- a/lib/galaxy_utils/sequence/vcf.py +++ b/lib/galaxy_utils/sequence/vcf.py @@ -8,12 +8,12 @@ class VariantCall( object ): header_startswith = None required_header_fields = None required_header_length = None - + @classmethod def get_class_by_format( cls, format ): assert format in VCF_FORMATS, 'Unknown format type specified: %s' % format return VCF_FORMATS[ format ] - + def __init__( self, vcf_line, metadata, sample_names ): raise Exception( 'Abstract Method' ) @@ -22,7 +22,7 @@ class VariantCall33( VariantCall ): header_startswith = '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO' required_header_fields = header_startswith.split( '\t' ) required_header_length = len( required_header_fields ) - + def __init__( self, vcf_line, metadata, sample_names ): # Raw line is needed for indexing file. self.raw_line = vcf_line @@ -31,7 +31,7 @@ class VariantCall33( VariantCall ): self.sample_names = sample_names self.format = None self.sample_values = [] - + #parse line self.fields = self.line.split( '\t' ) if sample_names: @@ -71,7 +71,7 @@ class Reader( object ): self.metadata_len = 0 self.sample_names = [] self.vcf_class = None - + # Read file metadata. while True: line = self.vcf_file.readline() diff --git a/lib/mimeparse.py b/lib/mimeparse.py index f37c2ba83de..8826a5c9284 100755 --- a/lib/mimeparse.py +++ b/lib/mimeparse.py @@ -1,7 +1,7 @@ """MIME-Type Parser This module provides basic functions for handling mime-types. It can handle -matching mime-types against a list of media-ranges. See section 14.1 of +matching mime-types against a list of media-ranges. See section 14.1 of the HTTP specification [RFC 2616] for a complete explanation. http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1 @@ -11,7 +11,7 @@ Contents: - parse_media_range(): Media-ranges are mime-types with wild-cards and a 'q' quality parameter. - quality(): Determines the quality ('q') of a mime-type when compared against a list of media-ranges. - quality_parsed(): Just like quality() except the second parameter must be pre-parsed. - - best_match(): Choose the mime-type with the highest quality ('q') from a list of candidates. + - best_match(): Choose the mime-type with the highest quality ('q') from a list of candidates. """ __version__ = "0.1.2" @@ -50,7 +50,7 @@ def parse_media_range(range): ('application', '*', {'q', '0.5'}) - In addition this function also guarantees that there + In addition this function also guarantees that there is a value for 'q' in the params dictionary, filling it in with a proper default if necessary. """ @@ -62,14 +62,14 @@ def parse_media_range(range): return (type, subtype, params) def fitness_and_quality_parsed(mime_type, parsed_ranges): - """Find the best match for a given mime-type against - a list of media_ranges that have already been + """Find the best match for a given mime-type against + a list of media_ranges that have already been parsed by parse_media_range(). Returns a tuple of the fitness value and the value of the 'q' quality parameter of the best match, or (-1, 0) if no match was found. Just as for quality_parsed(), 'parsed_ranges' must be a list of parsed media ranges. """ - best_fitness = -1 + best_fitness = -1 best_fit_q = 0 (target_type, target_subtype, target_params) =\ parse_media_range(mime_type) @@ -85,7 +85,7 @@ def fitness_and_quality_parsed(mime_type, parsed_ranges): if fitness > best_fitness: best_fitness = fitness best_fit_q = params['q'] - + return best_fitness, float(best_fit_q) def quality_parsed(mime_type, parsed_ranges): @@ -104,18 +104,18 @@ def quality(mime_type, ranges): >>> quality('text/html','text/*;q=0.3, text/html;q=0.7, text/html;level=1, text/html;level=2;q=0.4, */*;q=0.5') 0.7 - - """ + + """ parsed_ranges = [parse_media_range(r) for r in ranges.split(",")] return quality_parsed(mime_type, parsed_ranges) def best_match(supported, header): """Takes a list of supported mime-types and finds the best match for all the media-ranges listed in header. The value of - header must be a string that conforms to the format of the + header must be a string that conforms to the format of the HTTP Accept: header. The value of 'supported' is a list of mime-types. - + >>> best_match(['application/xbel+xml', 'text/xml'], 'text/*;q=0.5,*/*; q=0.1') 'text/xml' """ @@ -178,7 +178,7 @@ if __name__ == "__main__": mime_types_supported = ['image/*', 'application/xml'] # match using a type wildcard self.assertEqual(best_match(mime_types_supported, 'image/png'), 'image/*') - # match using a wildcard for both requested and supported + # match using a wildcard for both requested and supported self.assertEqual(best_match(mime_types_supported, 'image/*'), 'image/*') - unittest.main() + unittest.main() diff --git a/lib/tool_shed/galaxy_install/__init__.py b/lib/tool_shed/galaxy_install/__init__.py index efc444471fd..908757cac49 100644 --- a/lib/tool_shed/galaxy_install/__init__.py +++ b/lib/tool_shed/galaxy_install/__init__.py @@ -52,4 +52,3 @@ class InstalledRepositoryManager( object ): tool_shed.util.datatype_util.load_installed_datatype_converters( self.app, installed_repository_dict, deactivate=deactivate ) if installed_repository_dict[ 'display_path' ]: tool_shed.util.datatype_util.load_installed_display_applications( self.app, installed_repository_dict, deactivate=deactivate ) - \ No newline at end of file diff --git a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py index 94309c5a5b2..06ca33a8d1c 100644 --- a/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py +++ b/lib/tool_shed/galaxy_install/grids/admin_toolshed_grids.py @@ -109,7 +109,7 @@ class InstalledRepositoryGrid( grids.Grid ): visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 68c53dccc6d..5ce1ff085d2 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -71,7 +71,7 @@ class InstallManager( object ): latest_migration_script_number ) else: # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but - # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in + # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in # the database. tool_shed_accessible = True missing_tool_configs_dict = odict() @@ -249,7 +249,7 @@ class InstallManager( object ): found = True break if found: - break + break full_path = str( os.path.abspath( os.path.join( root, name ) ) ) tool = self.toolbox.load_tool( full_path ) return suc.generate_tool_guid( repository_clone_url, tool ) diff --git a/lib/tool_shed/galaxy_install/migrate/check.py b/lib/tool_shed/galaxy_install/migrate/check.py index 2ee86ffdf62..3486289f458 100644 --- a/lib/tool_shed/galaxy_install/migrate/check.py +++ b/lib/tool_shed/galaxy_install/migrate/check.py @@ -61,7 +61,7 @@ def verify_tools( app, url, galaxy_config_file, engine_options={} ): latest_tool_migration_script_number ) else: # It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but - # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in + # we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in # the database. tool_shed_accessible = True missing_tool_configs_dict = odict() diff --git a/lib/tool_shed/galaxy_install/migrate/versions/0005_tools.py b/lib/tool_shed/galaxy_install/migrate/versions/0005_tools.py index 19b6fa2bfd6..d6c2a856b8b 100644 --- a/lib/tool_shed/galaxy_install/migrate/versions/0005_tools.py +++ b/lib/tool_shed/galaxy_install/migrate/versions/0005_tools.py @@ -1,9 +1,9 @@ """ -The tools "Map with BWA for Illumina" and "Map with BWA for SOLiD" have -been eliminated from the distribution. The tools are now available +The tools "Map with BWA for Illumina" and "Map with BWA for SOLiD" have +been eliminated from the distribution. The tools are now available in the repository named bwa_wrappers from the main Galaxy tool shed at -http://toolshed.g2.bx.psu.edu, and will be installed into your local -Galaxy instance at the location discussed above by running the following +http://toolshed.g2.bx.psu.edu, and will be installed into your local +Galaxy instance at the location discussed above by running the following command. """ diff --git a/lib/tool_shed/galaxy_install/migrate/versions/0007_tools.py b/lib/tool_shed/galaxy_install/migrate/versions/0007_tools.py index 04b5eea5d55..934316ab361 100644 --- a/lib/tool_shed/galaxy_install/migrate/versions/0007_tools.py +++ b/lib/tool_shed/galaxy_install/migrate/versions/0007_tools.py @@ -1,11 +1,11 @@ """ The following tools have been eliminated from the distribution: -Map with Bowtie for Illumina, Map with Bowtie for SOLiD, Lastz, -and Lastz paired reads. The tools are now available in the -repositories named bowtie_wrappers, bowtie_color_wrappers, lastz, -and lastz_paired_reads from the main Galaxy tool shed at -http://toolshed.g2.bx.psu.edu, and will be installed into your -local Galaxy instance at the location discussed above by running +Map with Bowtie for Illumina, Map with Bowtie for SOLiD, Lastz, +and Lastz paired reads. The tools are now available in the +repositories named bowtie_wrappers, bowtie_color_wrappers, lastz, +and lastz_paired_reads from the main Galaxy tool shed at +http://toolshed.g2.bx.psu.edu, and will be installed into your +local Galaxy instance at the location discussed above by running the following command. """ diff --git a/lib/tool_shed/galaxy_install/repository_util.py b/lib/tool_shed/galaxy_install/repository_util.py index a28c17b7349..d37643c6244 100644 --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -35,8 +35,8 @@ def create_repo_info_dict( trans, repository_clone_url, changeset_revision, ctx_ repository_metadata=None, tool_dependencies=None, repository_dependencies=None ): """ Return a dictionary that includes all of the information needed to install a repository into a local Galaxy instance. The dictionary will also - contain the recursive list of repository dependencies defined for the repository, as well as the defined tool dependencies. - + contain the recursive list of repository dependencies defined for the repository, as well as the defined tool dependencies. + This method is called from Galaxy under three scenarios: 1. During the tool shed repository installation process via the tool shed's get_repository_information() method. In this case both the received repository and repository_metadata will be objects., but tool_dependencies and repository_dependencies will be None @@ -166,7 +166,7 @@ def get_repair_dict( trans, repository ): # The installed_repository must be in the uninstalled state. The structure of tool_panel_section_dict is: # { : [{ 'id':

    , 'name':
    , 'version':
    , 'tool_config': }]} # Here is an example: - # {"localhost:9009/repos/test/filter/Filter1/1.1.0": + # {"localhost:9009/repos/test/filter/Filter1/1.1.0": # [{"id": "filter_and_sort", "name": "Filter and Sort", "tool_config": "filtering.xml", "version": ""}]} # Currently all tools contained within an installed tool shed repository must be loaded into the same section in the tool panel, so we can # get the section id of the first guid in the tool_panel_section_dict. In the future, we'll have to handle different sections per guid. @@ -186,7 +186,7 @@ def get_repair_dict( trans, repository ): else: # The installed_repository must be in the installed state, so we can skip determining if it has tools that are displayed in a tool panel section # since no changes will be made to it. - tool_panel_section_keys.append( None ) + tool_panel_section_keys.append( None ) else: tool_dependencies = None # The tools will be loaded outside of any sections in the tool panel. @@ -292,7 +292,7 @@ def get_tool_shed_repository_ids( as_string=False, **kwd ): if as_string: '' return [] - + def get_update_to_changeset_revision_and_ctx_rev( trans, repository ): """Return the changeset revision hash to which the repository can be updated.""" changeset_revision_dict = {} @@ -502,7 +502,7 @@ def install_tool_shed_repository( trans, tool_shed_repository, repo_info_dict, t try: tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] except KeyError: - log.debug( 'Invalid tool_panel_section_key "%s" specified. Tools will be loaded outside of sections in the tool panel.', + log.debug( 'Invalid tool_panel_section_key "%s" specified. Tools will be loaded outside of sections in the tool panel.', str( tool_panel_section_key ) ) tool_section = None else: @@ -591,7 +591,7 @@ def merge_containers_dicts_for_new_install( containers_dicts ): dependencies and tool dependencies. The entries for missing dependencies are all None since they have previously been merged into the installed dependencies. This method will merge the dependencies entries into a single container and return it for display. """ - new_containers_dict = dict( readme_files=None, + new_containers_dict = dict( readme_files=None, datatypes=None, missing_repository_dependencies=None, repository_dependencies=None, diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py index f54698dccf6..37b088b22a4 100755 --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -63,44 +63,44 @@ def handle_environment_variables( app, tool_dependency, install_dir, env_var_dic """ This method works with with a combination of three tool dependency definition tag sets, which are defined in the tool_dependencies.xml file in the order discussed here. The example for this discussion is the tool_dependencies.xml file contained in the osra repository, which is available at: - - http://testtoolshed.g2.bx.psu.edu/view/bgruening/osra - + + http://testtoolshed.g2.bx.psu.edu/view/bgruening/osra + The first tag set defines a complex repository dependency like this. This tag set ensures that changeset revision XXX of the repository named package_graphicsmagick_1_3 owned by YYY in the tool shed ZZZ has been previously installed. - + ... - + * By the way, there is an env.sh file associated with version 1.3.18 of the graphicsmagick package which looks something like this (we'll reference this file later in this discussion. ---- - GRAPHICSMAGICK_ROOT_DIR=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick; + GRAPHICSMAGICK_ROOT_DIR=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick; export GRAPHICSMAGICK_ROOT_DIR ---- - + The second tag set defines a specific package dependency that has been previously installed (guaranteed by the tag set discussed above) and compiled, where the compiled dependency is needed by the tool dependency currently being installed (osra version 2.0.0 in this case) and complied in order for it's installation and compilation to succeed. This tag set is contained within the tag set, which implies that version 2.0.0 of the osra package requires version 1.3.18 of the graphicsmagick package in order to successfully compile. When this tag set is handled, one of the effects is that the env.sh file associated with graphicsmagick version 1.3.18 is "sourced", which undoubtedly sets or alters certain environment variables (e.g. PATH, PYTHONPATH, etc). - + - + The third tag set enables discovery of the same required package dependency discussed above for correctly compiling the osra version 2.0.0 package, but in this case the package can be discovered at tool execution time. Using the $ENV[] option as shown in this example, the value of the environment variable named GRAPHICSMAGICK_ROOT_DIR (which was set in the environment using the second tag set described above) will be used to automatically alter the env.sh file associated with the osra version 2.0.0 tool dependency when it is installed into Galaxy. * Refer to where we discussed the env.sh file - for version 1.3.18 of the graphicsmagick package above. + for version 1.3.18 of the graphicsmagick package above. $ENV[GRAPHICSMAGICK_ROOT_DIR]/lib/ @@ -112,7 +112,7 @@ def handle_environment_variables( app, tool_dependency, install_dir, env_var_dic The above tag will produce an env.sh file for version 2.0.0 of the osra package when it it installed into Galaxy that looks something like this. Notice that the path to the gmagick binary is included here since it expands the defined $ENV[GRAPHICSMAGICK_ROOT_DIR] value in the above tag set. - + ---- LD_LIBRARY_PATH=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH; export LD_LIBRARY_PATH diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py index d2548b7330d..e644c458030 100755 --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -98,7 +98,7 @@ def get_tool_shed_repository_install_dir( app, tool_shed_repository ): def get_updated_changeset_revisions_from_tool_shed( app, tool_shed_url, name, owner, changeset_revision ): """ - Get all appropriate newer changeset revisions for the repository defined by + Get all appropriate newer changeset revisions for the repository defined by the received tool_shed_url / name / owner combination. """ url = suc.url_join( tool_shed_url, @@ -161,7 +161,7 @@ def handle_set_environment_entry_for_package( app, install_dir, tool_shed_reposi # setting in the received tool_shed_repository's tool_dependencies.xml file and the required repository's # tool_dependencies.xml file may include the use of the $ENV[] variable inheritance feature. To handle this, # we will replace the current "value" entries in each env_var_dict with the actual path taken from the env.sh - # file generated for the installed required repository. Each env_var_dict currently looks something like this: + # file generated for the installed required repository. Each env_var_dict currently looks something like this: # {'action': 'append_to', 'name': 'LD_LIBRARY_PATH', 'value': '$BOOST_ROOT_DIR/lib/'} # We'll read the contents of the received required_repository's env.sh file and replace the 'value' entry of each env_var_dict # with the associated value in the env.sh file. @@ -203,7 +203,7 @@ def install_and_build_package_via_fabric( app, tool_dependency, actions_dict ): except Exception, e: log.exception( 'Error installing tool dependency %s version %s.', str( tool_dependency.name ), str( tool_dependency.version ) ) tool_dependency.status = app.model.ToolDependency.installation_status.ERROR - tool_dependency.error_message = '%s\n%s' % ( td_common_util.format_traceback(), str( e ) ) + tool_dependency.error_message = '%s\n%s' % ( td_common_util.format_traceback(), str( e ) ) sa_session.add( tool_dependency ) sa_session.flush() if tool_dependency.status != app.model.ToolDependency.installation_status.ERROR: @@ -255,7 +255,7 @@ def install_package( app, elem, tool_shed_repository, tool_dependencies=None ): # Set this dependent repository's tool dependency env.sh file with a path to the required repository's installed tool dependency package. # We can get everything we need from the discovered installed required_repository. if required_repository.status in [ app.model.ToolShedRepository.installation_status.DEACTIVATED, - app.model.ToolShedRepository.installation_status.INSTALLED ]: + app.model.ToolShedRepository.installation_status.INSTALLED ]: if not os.path.exists( required_repository_package_install_dir ): print 'Missing required tool dependency directory %s' % str( required_repository_package_install_dir ) repo_files_dir = required_repository.repo_files_directory( app ) @@ -484,7 +484,7 @@ def install_via_fabric( app, tool_dependency, actions_elem, install_dir, package # # # This action type allows for defining an environment that will properly compile a tool dependency. Currently, tag set definitions like - # that above are supported, but in the future other approaches to setting environment variables or other environment attributes can be + # that above are supported, but in the future other approaches to setting environment variables or other environment attributes can be # supported. The above tag set will result in the installed and compiled numpy version 1.7.1 binary to be used when compiling the current # tool dependency package. See the package_matplotlib_1_2 repository in the test tool shed for a real-world example. for env_elem in action_elem: @@ -574,7 +574,7 @@ def parse_env_shell_entry( action, name, value, line ): new_value = new_value.split( ';' )[ 0 ] return new_value - + def populate_actions_dict( app, dependent_install_dir, required_install_dir, tool_shed_repository, required_repository, package_name, package_version, tool_dependencies_config ): """ Populate an actions dictionary that can be sent to fabric_util.install_and_build_package. This method handles the scenario where a tool_dependencies.xml @@ -648,7 +648,7 @@ def run_proprietary_fabric_method( app, elem, proprietary_fabfile_path, install_ except Exception, e: return "Exception executing fabric script %s: %s. " % ( str( proprietary_fabfile_path ), str( e ) ) if returncode: - return message + return message handle_environment_settings( app, tool_dependency, install_dir, cmd ) def run_subprocess( app, cmd ): diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py index 63eb0781ae9..1dca623ebcb 100755 --- a/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/td_common_util.py @@ -295,7 +295,7 @@ def zip_extraction_directory( file_path, file_name ): def zipfile_ok( path_to_archive ): """ - This function is a bit pedantic and not functionally necessary. It checks whether there is no file pointing outside of the extraction, + This function is a bit pedantic and not functionally necessary. It checks whether there is no file pointing outside of the extraction, because ZipFile.extractall() has some potential security holes. See python zipfile documentation for more details. """ basename = os.path.realpath( os.path.dirname( path_to_archive ) ) diff --git a/lib/tool_shed/grids/admin_grids.py b/lib/tool_shed/grids/admin_grids.py index 26146e53c99..a14774ead83 100644 --- a/lib/tool_shed/grids/admin_grids.py +++ b/lib/tool_shed/grids/admin_grids.py @@ -81,8 +81,8 @@ class UserGrid( grids.Grid ): key="email", visible=False ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -168,8 +168,8 @@ class RoleGrid( grids.Grid ): visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1], columns[2] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -241,8 +241,8 @@ class GroupGrid( grids.Grid ): visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", - cols_to_filter=[ columns[0], columns[1], columns[2] ], + columns.append( grids.MulticolFilterColumn( "Search", + cols_to_filter=[ columns[0], columns[1], columns[2] ], key="free-text-search", visible=False, filterable="standard" ) ) @@ -306,7 +306,7 @@ class AdminRepositoryGrid( RepositoryGrid ): RepositoryGrid.DeprecatedColumn( "Deprecated", key="deprecated", attach_popup=False ), # Columns that are valid for filtering but are not visible. DeletedColumn( "Deleted", key="deleted", attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, @@ -421,7 +421,7 @@ class RepositoryMetadataGrid( grids.Grid ): DeprecatedColumn( "Deprecated", attach_popup=False ), MaliciousColumn( "Malicious", attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[1] ], key="free-text-search", visible=False, diff --git a/lib/tool_shed/grids/repository_grids.py b/lib/tool_shed/grids/repository_grids.py index 8870d1914fe..4a054851f6c 100644 --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -235,7 +235,7 @@ class RepositoryGrid( grids.Grid ): key="Category.name", visible=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, description", + columns.append( grids.MulticolFilterColumn( "Search repository name, description", cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, @@ -382,7 +382,7 @@ class MyWritableRepositoriesGrid( RepositoryGrid ): attach_popup=False, key="User.username" ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[ 0 ] ], key="free-text-search", visible=False, @@ -472,7 +472,7 @@ class RepositoriesInCategoryGrid( RepositoryGrid ): key="email", visible=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, description", + columns.append( grids.MulticolFilterColumn( "Search repository name, description", cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, @@ -512,7 +512,7 @@ class RepositoriesIOwnGrid( RepositoryGrid ): RepositoryGrid.ToolsFunctionallyCorrectColumn( "Tools
    Verified" ), RepositoryGrid.DeprecatedColumn( "Deprecated" ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, @@ -544,7 +544,7 @@ class RepositoriesMissingToolTestComponentsGrid( RepositoryGrid ): link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, @@ -631,7 +631,7 @@ class RepositoriesWithTestInstallErrorsGrid( RepositoryGrid ): link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, @@ -718,7 +718,7 @@ class RepositoriesWithSkipTestsCheckedGrid( RepositoryGrid ): link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, @@ -805,7 +805,7 @@ class DeprecatedRepositoriesIOwnGrid( RepositoriesIOwnGrid ): key="Category.name", attach_popup=False ), ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, @@ -837,7 +837,7 @@ class RepositoriesWithFailingToolTestsGrid( RepositoryGrid ): link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, @@ -924,7 +924,7 @@ class RepositoriesWithNoFailingToolTestsGrid( RepositoryGrid ): link=( lambda item: dict( operation="repositories_by_user", id=item.id ) ), attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, @@ -1023,7 +1023,7 @@ class RepositoriesWithInvalidToolsGrid( RepositoryGrid ): return val title = "Repositories with invalid tools" - columns = [ + columns = [ InvalidToolConfigColumn( "Tool config" ), RepositoryGrid.NameColumn( "Name", key="name", @@ -1228,7 +1228,7 @@ class RepositoryMetadataGrid( grids.Grid ): attach_popup=False, key="User.username" ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, description", + columns.append( grids.MulticolFilterColumn( "Search repository name, description", cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, @@ -1308,7 +1308,7 @@ class RepositoryDependenciesGrid( RepositoryMetadataGrid ): RepositoryMetadataGrid.ChangesetRevisionColumn( "Revision", attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, owner", + columns.append( grids.MulticolFilterColumn( "Search repository name, owner", cols_to_filter=[ columns[1], columns[2] ], key="free-text-search", visible=False, @@ -1375,7 +1375,7 @@ class DatatypesGrid( RepositoryMetadataGrid ): RepositoryMetadataGrid.ChangesetRevisionColumn( "Revision", attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, owner", + columns.append( grids.MulticolFilterColumn( "Search repository name, owner", cols_to_filter=[ columns[1], columns[2] ], key="free-text-search", visible=False, @@ -1450,7 +1450,7 @@ class ToolDependenciesGrid( RepositoryMetadataGrid ): RepositoryMetadataGrid.ChangesetRevisionColumn( "Revision", attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, owner", + columns.append( grids.MulticolFilterColumn( "Search repository name, owner", cols_to_filter=[ columns[1], columns[2] ], key="free-text-search", visible=False, @@ -1514,7 +1514,7 @@ class ToolsGrid( RepositoryMetadataGrid ): RepositoryMetadataGrid.ChangesetRevisionColumn( "Revision", attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, owner", + columns.append( grids.MulticolFilterColumn( "Search repository name, owner", cols_to_filter=[ columns[1], columns[2] ], key="free-text-search", visible=False, @@ -1632,7 +1632,7 @@ class ValidRepositoryGrid( RepositoryGrid ): key="Category.name", visible=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, description", + columns.append( grids.MulticolFilterColumn( "Search repository name, description", cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, diff --git a/lib/tool_shed/grids/repository_review_grids.py b/lib/tool_shed/grids/repository_review_grids.py index 7870eb5c324..99db9039b19 100644 --- a/lib/tool_shed/grids/repository_review_grids.py +++ b/lib/tool_shed/grids/repository_review_grids.py @@ -131,7 +131,7 @@ class RepositoriesWithReviewsGrid( RepositoryGrid ): if review.approved: rval += '%s
    ' % review.approved return rval - + title = "All reviewed repositories" model_class = model.Repository template='/webapps/tool_shed/repository_review/grid.mako' @@ -150,12 +150,12 @@ class RepositoriesWithReviewsGrid( RepositoryGrid ): RatingColumn( "Rating", attach_popup=False ), ApprovedColumn( "Approved", attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[ 0 ] ], key="free-text-search", visible=False, filterable="standard" ) ) - operations = [ + operations = [ grids.GridOperation( "Inspect repository revisions", allow_multiple=False, condition=( lambda item: not item.deleted ), @@ -189,7 +189,7 @@ class RepositoriesWithoutReviewsGrid( RepositoriesWithReviewsGrid ): attach_popup=False, key="User.username" ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, description", + columns.append( grids.MulticolFilterColumn( "Search repository name, description", cols_to_filter=[ columns[ 0 ], columns[ 1 ] ], key="free-text-search", visible=False, @@ -227,7 +227,7 @@ class RepositoriesReadyForReviewGrid( RepositoriesWithoutReviewsGrid ): attach_popup=False, key="User.username" ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, description", + columns.append( grids.MulticolFilterColumn( "Search repository name, description", cols_to_filter=[ columns[ 0 ], columns[ 1 ] ], key="free-text-search", visible=False, @@ -264,7 +264,7 @@ class RepositoriesReviewedByMeGrid( RepositoriesWithReviewsGrid ): RepositoriesWithReviewsGrid.RatingColumn( "Rating", attach_popup=False ), RepositoriesWithReviewsGrid.ApprovedColumn( "Approved", attach_popup=False ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[ 0 ] ], key="free-text-search", visible=False, @@ -347,7 +347,7 @@ class RepositoryReviewsByUserGrid( grids.Grid ): # Override these default_filter = {} global_actions = [] - operations = [ + operations = [ grids.GridOperation( "Inspect repository revisions", allow_multiple=False, condition=( lambda item: not item.deleted ), @@ -379,12 +379,12 @@ class ReviewedRepositoriesIOwnGrid( RepositoriesWithReviewsGrid ): RepositoriesWithReviewsGrid.ReviewersColumn( "Reviewers", attach_popup=False ), RepositoryGrid.DeprecatedColumn( "Deprecated" ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name", + columns.append( grids.MulticolFilterColumn( "Search repository name", cols_to_filter=[ columns[0] ], key="free-text-search", visible=False, filterable="standard" ) ) - operations = [ + operations = [ grids.GridOperation( "Inspect repository revisions", allow_multiple=False, condition=( lambda item: not item.deleted ), @@ -420,7 +420,7 @@ class RepositoriesWithNoToolTestsGrid( RepositoriesWithoutReviewsGrid ): attach_popup=False, key="User.username" ) ] - columns.append( grids.MulticolFilterColumn( "Search repository name, description", + columns.append( grids.MulticolFilterColumn( "Search repository name, description", cols_to_filter=[ columns[ 0 ], columns[ 1 ] ], key="free-text-search", visible=False, diff --git a/lib/tool_shed/repository_types/tool_dependency_definition.py b/lib/tool_shed/repository_types/tool_dependency_definition.py index 2f901c94659..4039b20796f 100644 --- a/lib/tool_shed/repository_types/tool_dependency_definition.py +++ b/lib/tool_shed/repository_types/tool_dependency_definition.py @@ -38,4 +38,3 @@ class ToolDependencyDefinition( TipOnly ): if file_name not in self.valid_file_names: return False return True - \ No newline at end of file diff --git a/lib/tool_shed/scripts/api/common.py b/lib/tool_shed/scripts/api/common.py index 9339a96c09a..b069c7ceb9b 100644 --- a/lib/tool_shed/scripts/api/common.py +++ b/lib/tool_shed/scripts/api/common.py @@ -20,7 +20,7 @@ def encode_id( config_id_secret, obj_id ): id_cipher = Blowfish.new( config_id_secret ) # Convert to string s = str( obj_id ) - # Pad to a multiple of 8 with leading "!" + # Pad to a multiple of 8 with leading "!" s = ( "!" * ( 8 - len(s) % 8 ) ) + s # Encrypt return id_cipher.encrypt( s ).encode( 'hex' ) diff --git a/lib/tool_shed/scripts/api/export.py b/lib/tool_shed/scripts/api/export.py index 1bc701b7fbc..7783d41fb07 100644 --- a/lib/tool_shed/scripts/api/export.py +++ b/lib/tool_shed/scripts/api/export.py @@ -33,7 +33,7 @@ def string_as_bool( string ): return True else: return False - + def main( options ): """Collect all user data and export the repository via the Tool Shed API.""" base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) diff --git a/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py b/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py index 0e82b33a5de..16a38bd65b2 100755 --- a/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py +++ b/lib/tool_shed/scripts/api/tool_shed_repository_revision_update.py @@ -2,7 +2,7 @@ """ PUT/update script to update appropriate values in a repository_metadata table record in the Tool Shed. -usage: tool_shed_repository_revision_update.py key url key1=value1 key2=value2 ... +usage: tool_shed_repository_revision_update.py key url key1=value1 key2=value2 ... """ import os, sys @@ -18,7 +18,7 @@ to_json_string = simplejson.dumps from_json_string = simplejson.loads data = {} -for key, value in [ kwarg.split( '=', 1 ) for kwarg in sys.argv[ 3: ] ]: +for key, value in [ kwarg.split( '=', 1 ) for kwarg in sys.argv[ 3: ] ]: """ This example script will properly handle updating the value of one or more of the following RepositoryMetadata attributes: tools_functionally_correct, do_not_test, tool_test_results diff --git a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py index b317e8c8116..548a4197e94 100644 --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -11,7 +11,7 @@ log.setLevel( 10 ) log.addHandler( logging.StreamHandler( sys.stdout ) ) from galaxy import eggs -import pkg_resources +import pkg_resources eggs.require( "SQLAlchemy >= 0.4" ) eggs.require( 'mercurial' ) from mercurial import hg, ui, commands, __version__ @@ -38,10 +38,10 @@ def main(): '''Script that checks repositories to see if the tools contained within them have functional tests defined.''' parser = OptionParser() parser.add_option( "-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False ) - parser.add_option( "-s", - "--section", - action="store", - dest="section", + parser.add_option( "-s", + "--section", + action="store", + dest="section", default='server:main', help="which .ini file section to extract the host and port from" ) parser.add_option( @@ -66,24 +66,24 @@ def main(): for key, value in config_parser.items( "app:main" ): config_dict[key] = value config = tool_shed_config.Configuration( **config_dict ) - + config_section = options.section now = strftime( "%Y-%m-%d %H:%M:%S" ) print "#############################################################################" print "# %s - Checking repositories for tools with functional tests." % now print "# This tool shed is configured to listen on %s:%s." % ( config_parser.get( config_section, 'host' ), config_parser.get( config_section, 'port' ) ) app = FlagRepositoriesApplication( config ) - + if options.info_only: print "# Displaying info only ( --info_only )" if options.verbosity: print "# Displaying extra information ( --verbosity = %d )" % options.verbosity - + check_and_flag_repositories( app, info_only=options.info_only, verbosity=options.verbosity ) def check_and_flag_repositories( app, info_only=False, verbosity=1 ): ''' - This method will iterate through all records in the repository_metadata table, checking each one for tool metadata, + This method will iterate through all records in the repository_metadata table, checking each one for tool metadata, then checking the tool metadata for tests. Each tool's metadata should look something like: { @@ -111,14 +111,14 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): "version": "1.2.3", "version_string_cmd": null } - + If the "tests" attribute is missing or empty, this script will mark the metadata record (which is specific to a changeset revision of a repository) not to be tested. If each "tools" attribute has at least one valid "tests" entry, this script will do nothing, and leave it available for the install and test repositories script to process. If the tested changeset revision does not have a test-data directory, this script will also mark the revision not to be tested. - + TODO: Update this dict structure with the recently added components. - + If any error is encountered, the script will update the repository_metadata.tool_test_results attribute following this structure: { "test_environment": @@ -155,8 +155,8 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): 'tool_dependencies': [ { - 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', - 'name': 'Name of the tool dependency.', + 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', + 'name': 'Name of the tool dependency.', 'version': 'Version if this is a package, otherwise blank.', 'error_message': 'The error message returned when installation was attempted.', }, @@ -164,8 +164,8 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): 'repository_dependencies': [ { - 'tool_shed': 'The tool shed that this repository was installed from.', - 'name': 'The name of the repository that failed to install.', + 'tool_shed': 'The tool shed that this repository was installed from.', + 'name': 'The name of the repository that failed to install.', 'owner': 'Owner of the failed repository.', 'changeset_revision': 'Changeset revision of the failed repository.', 'error_message': 'The error message that was returned when the repository failed to install.', @@ -174,8 +174,8 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): 'current_repository': [ { - 'tool_shed': 'The tool shed that this repository was installed from.', - 'name': 'The name of the repository that failed to install.', + 'tool_shed': 'The tool shed that this repository was installed from.', + 'name': 'The name of the repository that failed to install.', 'owner': 'Owner of the failed repository.', 'changeset_revision': 'Changeset revision of the failed repository.', 'error_message': 'The error message that was returned when the repository failed to install.', @@ -220,7 +220,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): app.model.RepositoryMetadata.table.c.do_not_test == False, not_( app.model.RepositoryMetadata.table.c.id.in_( skip_metadata_ids ) ) ) ): records_checked += 1 - # Initialize the repository_status dict with the test environment, but leave the test_errors empty. + # Initialize the repository_status dict with the test environment, but leave the test_errors empty. repository_status = {} if metadata_record.tool_test_results: repository_status = metadata_record.tool_test_results @@ -241,7 +241,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): checked_repository_ids.append( metadata_record.repository.id ) if verbosity >= 1: print '# -------------------------------------------------------------------------------------------' - print '# Now checking revision %s of %s, owned by %s.' % ( changeset_revision, name, owner ) + print '# Now checking revision %s of %s, owned by %s.' % ( changeset_revision, name, owner ) # If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning # only repositories that contain tools. if 'tools' not in metadata_record.metadata: @@ -271,7 +271,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): else: print '# Test data directory found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) print '# Checking for functional tests in changeset revision %s of %s, owned by %s.' % \ - ( changeset_revision, name, owner ) + ( changeset_revision, name, owner ) # Loop through all the tools in this metadata record, checking each one for defined functional tests. for tool_metadata in metadata_record.metadata[ 'tools' ]: tool_count += 1 @@ -280,7 +280,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): tool_guid = tool_metadata[ 'guid' ] if verbosity >= 2: print "# Checking tool ID '%s' in changeset revision %s of %s." % \ - ( tool_id, changeset_revision, name ) + ( tool_id, changeset_revision, name ) # If there are no tests, this tool should not be tested, since the tool functional tests only report failure if the test itself fails, # not if it's missing or undefined. Filtering out those repositories at this step will reduce the number of "false negatives" the # automated functional test framework produces. @@ -294,7 +294,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): tool_has_tests = True if verbosity >= 2: print "# Tool ID '%s' in changeset revision %s of %s has one or more valid functional tests defined." % \ - ( tool_id, changeset_revision, name ) + ( tool_id, changeset_revision, name ) has_tests += 1 failure_reason = '' problem_found = False @@ -305,7 +305,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): if missing_test_files: if verbosity >= 2: print "# Tool ID '%s' in changeset revision %s of %s is missing one or more required test files: %s" % \ - ( tool_id, changeset_revision, name, ', '.join( missing_test_files ) ) + ( tool_id, changeset_revision, name, ', '.join( missing_test_files ) ) else: has_test_files = True if not has_test_data: @@ -355,8 +355,8 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): # 'tool_dependencies': # [ # { - # 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', - # 'name': 'Name of the tool dependency.', + # 'type': 'Type of tool dependency, e.g. package, set_environment, etc.', + # 'name': 'Name of the tool dependency.', # 'version': 'Version if this is a package, otherwise blank.', # 'error_message': 'The error message returned when installation was attempted.', # }, @@ -364,8 +364,8 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): # 'repository_dependencies': # [ # { - # 'tool_shed': 'The tool shed that this repository was installed from.', - # 'name': 'The name of the repository that failed to install.', + # 'tool_shed': 'The tool shed that this repository was installed from.', + # 'name': 'The name of the repository that failed to install.', # 'owner': 'Owner of the failed repository.', # 'changeset_revision': 'Changeset revision of the failed repository.', # 'error_message': 'The error message that was returned when the repository failed to install.', @@ -374,8 +374,8 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): # 'current_repository': # [ # { - # 'tool_shed': 'The tool shed that this repository was installed from.', - # 'name': 'The name of the repository that failed to install.', + # 'tool_shed': 'The tool shed that this repository was installed from.', + # 'name': 'The name of the repository that failed to install.', # 'owner': 'Owner of the failed repository.', # 'changeset_revision': 'Changeset revision of the failed repository.', # 'error_message': 'The error message that was returned when the repository failed to install.', @@ -398,7 +398,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): # }, # ] # } - # + # # Optionally, "traceback" may be included in a test_errors dict, if it is relevant. No script should overwrite anything other # than the list relevant to what it is testing. # Only append this error dict if it hasn't already been added. @@ -426,7 +426,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): # If repository_status[ 'test_errors' ] is empty, no issues were found, and we can just update time_last_tested with the platform # on which this script was run. if missing_test_components: - # If functional test definitions or test data are missing, set do_not_test = True if no tool with valid tests has been + # If functional test definitions or test data are missing, set do_not_test = True if no tool with valid tests has been # found in this revision, and: # a) There are multiple downloadable revisions, and the revision being tested is not the most recent downloadable revision. # In this case, the revision will never be updated with the missing components, and re-testing it would be redundant. @@ -455,7 +455,7 @@ def check_and_flag_repositories( app, info_only=False, verbosity=1 ): if info_only: print '# Database not updated, info_only set.' print "# Elapsed time: ", stop - start - print "#############################################################################" + print "#############################################################################" def get_repo_changelog_tuples( repo_path ): repo = hg.repository( ui.ui(), repo_path ) @@ -498,7 +498,7 @@ def should_set_do_not_test_flag( app, repository, changeset_revision ): Returns True if: a) There are multiple downloadable revisions, and the provided changeset revision is not the most recent downloadable revision. In this case, the revision will never be updated with correct data, and re-testing it would be redundant. - b) There are one or more downloadable revisions, and the provided changeset revision is the most recent downloadable revision. In this case, if + b) There are one or more downloadable revisions, and the provided changeset revision is the most recent downloadable revision. In this case, if the repository is updated with test data or functional tests, the downloadable changeset revision that was tested will either be replaced with the new changeset revision, or a new downloadable changeset revision will be created, either of which will be automatically checked and flagged as appropriate. In the install and test script, this behavior is slightly different, since we do want to always run functional tests @@ -518,8 +518,8 @@ def should_set_do_not_test_flag( app, repository, changeset_revision ): return True else: return False - - + + class FlagRepositoriesApplication( object ): """Encapsulates the state of a Universe application""" def __init__( self, config ): diff --git a/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py b/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py index f1a4464e114..b78a819c09e 100644 --- a/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py +++ b/lib/tool_shed/scripts/deprecate_repositories_without_metadata.py @@ -11,7 +11,7 @@ log.setLevel( 10 ) log.addHandler( logging.StreamHandler( sys.stdout ) ) from galaxy import eggs -import pkg_resources +import pkg_resources pkg_resources.require( "SQLAlchemy >= 0.4" ) import time, ConfigParser, shutil @@ -48,16 +48,16 @@ def main(): for key, value in config_parser.items( "app:main" ): config_dict[key] = value config = tool_shed_config.Configuration( **config_dict ) - + app = DeprecateRepositoriesApplication( config ) cutoff_time = datetime.utcnow() - timedelta( days=options.days ) now = strftime( "%Y-%m-%d %H:%M:%S" ) print "\n####################################################################################" print "# %s - Handling stuff older than %i days" % ( now, options.days ) - + if options.info_only: print "# Displaying info only ( --info_only )" - + deprecate_repositories( app, cutoff_time, days=options.days, info_only=options.info_only, verbose=options.verbose ) def send_mail_to_owner( app, name, owner, email, repositories_deprecated, days=14 ): @@ -102,7 +102,7 @@ def deprecate_repositories( app, cutoff_time, days=14, info_only=False, verbose= repository_ids_to_not_check = [] # Get a unique list of repository ids from the repository_metadata table. Any repository ID found in this table is not # empty, and will not be checked. - metadata_records = sa.select( [ distinct( app.model.RepositoryMetadata.table.c.repository_id ) ], + metadata_records = sa.select( [ distinct( app.model.RepositoryMetadata.table.c.repository_id ) ], from_obj=app.model.RepositoryMetadata.table ) \ .execute() for metadata_record in metadata_records: @@ -145,7 +145,7 @@ def deprecate_repositories( app, cutoff_time, days=14, info_only=False, verbose= stop = time.time() print '# Deprecated %d repositories.' % len( repositories ) print "# Elapsed time: ", stop - start - print "####################################################################################" + print "####################################################################################" class DeprecateRepositoriesApplication( object ): """Encapsulates the state of a Universe application""" diff --git a/lib/tool_shed/tool_shed_registry.py b/lib/tool_shed/tool_shed_registry.py index ca174814713..2d17e591b54 100644 --- a/lib/tool_shed/tool_shed_registry.py +++ b/lib/tool_shed/tool_shed_registry.py @@ -39,7 +39,7 @@ class Registry( object ): def password_manager_for_url( self, url ): """ - If the tool shed is using external auth, the client to the tool shed must authenticate to that as well. This provides access to the + If the tool shed is using external auth, the client to the tool shed must authenticate to that as well. This provides access to the urllib2.HTTPPasswordMgrWithdefaultRealm() object for the url passed in. Following more what galaxy.demo_sequencer.controllers.common does might be more appropriate at some stage... diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index feb7d3e3244..89d8db453df 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -36,7 +36,7 @@ def bundle_to_json( fh ): hg_unbundle10_obj = readbundle( fh, None ) groups = [ group for group in unpack_groups( hg_unbundle10_obj ) ] return json.to_json_string( groups, indent=4 ) - + def check_archive( repository, archive ): for member in archive.getmembers(): # Allow regular files and directories only @@ -134,7 +134,7 @@ def handle_bz2( repository, uploaded_file_name ): shutil.move( uncompressed, uploaded_file_name ) def handle_directory_changes( trans, repository, full_path, filenames_in_archive, remove_repo_files_not_in_tar, new_repo_alert, commit_message, - undesirable_dirs_removed, undesirable_files_removed ): + undesirable_dirs_removed, undesirable_files_removed ): repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) content_alert_str = '' @@ -210,7 +210,7 @@ def handle_missing_repository_attribute( elem ): error_message += 'The tag is missing the required owner attribute. ' log.debug( error_message ) return error_message - + def handle_gzip( repository, uploaded_file_name ): fd, uncompressed = tempfile.mkstemp( prefix='repo_%d_upload_gunzip_' % repository.id, dir=os.path.dirname( uploaded_file_name ), text=False ) gzipped_file = gzip.GzipFile( uploaded_file_name, 'rb' ) @@ -306,7 +306,7 @@ def handle_tool_dependencies_definition( trans, tool_dependencies_config, unpopu tree, error_message = xml_util.parse_xml( tool_dependencies_config ) if tree is None: return False, None - root = tree.getroot() + root = tree.getroot() if root.tag == 'tool_dependency': for root_index, root_elem in enumerate( root ): # diff --git a/lib/tool_shed/util/common_install_util.py b/lib/tool_shed/util/common_install_util.py index e4b586699e6..19ee52f37cd 100644 --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -88,7 +88,7 @@ def get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, inclu has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False ) includes_tools_for_display_in_tool_panel = all_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False ) includes_tool_dependencies = all_repo_info_dict.get( 'includes_tool_dependencies', False ) - includes_tools = all_repo_info_dict.get( 'includes_tools', False ) + includes_tools = all_repo_info_dict.get( 'includes_tools', False ) required_repo_info_dicts = all_repo_info_dict.get( 'all_repo_info_dicts', [] ) # Display tool dependencies defined for each of the repository dependencies. if required_repo_info_dicts: @@ -131,7 +131,7 @@ def get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, inclu has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False ) includes_tools_for_display_in_tool_panel = all_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False ) includes_tool_dependencies = all_repo_info_dict.get( 'includes_tool_dependencies', False ) - includes_tools = all_repo_info_dict.get( 'includes_tools', False ) + includes_tools = all_repo_info_dict.get( 'includes_tools', False ) required_repo_info_dicts = all_repo_info_dict.get( 'all_repo_info_dicts', [] ) installed_rd = None missing_rd = None @@ -188,7 +188,7 @@ def get_installed_and_missing_repository_dependencies( trans, repository ): if missing_rd_tups: missing_repository_dependencies[ 'root_key' ] = root_key missing_repository_dependencies[ root_key ] = missing_rd_tups - missing_repository_dependencies[ 'description' ] = description + missing_repository_dependencies[ 'description' ] = description return installed_repository_dependencies, missing_repository_dependencies def get_installed_and_missing_repository_dependencies_for_new_install( trans, repo_info_tuple ): @@ -333,13 +333,13 @@ def get_required_repo_info_dicts( trans, tool_shed_url, repo_info_dicts ): else: if v and not all_required_repo_info_dict[ k ]: all_required_repo_info_dict[ k ] = v - if required_repo_info_dicts: + if required_repo_info_dicts: for required_repo_info_dict in required_repo_info_dicts: if required_repo_info_dict not in all_repo_info_dicts: all_repo_info_dicts.append( required_repo_info_dict ) all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts return all_required_repo_info_dict - + def handle_tool_dependencies( app, tool_shed_repository, tool_dependencies_config, tool_dependencies ): """ Install and build tool dependencies defined in the tool_dependencies_config. This config's tag sets can currently refer to installation diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index b11ba2fa298..59aadcd62f7 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -100,7 +100,7 @@ def get_non_shed_tool_panel_configs( app ): # tree, error_message = xml_util.parse_xml( config_filename ) if tree is None: - continue + continue root = tree.getroot() tool_path = root.get( 'tool_path', None ) if tool_path is None: diff --git a/lib/tool_shed/util/container_util.py b/lib/tool_shed/util/container_util.py index aba54740ac6..ddcc583a6cb 100644 --- a/lib/tool_shed/util/container_util.py +++ b/lib/tool_shed/util/container_util.py @@ -376,12 +376,12 @@ def build_datatypes_folder( trans, folder_id, datatypes, label='Datatypes' ): subclass='subclass' ) folder.datatypes.append( datatype ) for datatypes_dict in datatypes: - # {"converters": - # [{"target_datatype": "gff", - # "tool_config": "bed_to_gff_converter.xml", - # "guid": "localhost:9009/repos/test/bed_to_gff_converter/CONVERTER_bed_to_gff_0/2.0.0"}], - # "display_in_upload": "true", - # "dtype": "galaxy.datatypes.interval:Bed", + # {"converters": + # [{"target_datatype": "gff", + # "tool_config": "bed_to_gff_converter.xml", + # "guid": "localhost:9009/repos/test/bed_to_gff_converter/CONVERTER_bed_to_gff_0/2.0.0"}], + # "display_in_upload": "true", + # "dtype": "galaxy.datatypes.interval:Bed", # "extension": "bed"} # TODO: converters and display_app information is not currently rendered. Should it be? # Handle defined converters, if any. @@ -1112,15 +1112,15 @@ def build_tool_test_results_folder( trans, folder_id, tool_test_results_dict, la tool_dependency_errors = installation_error_dicts.get( 'tool_dependencies', [] ) if current_repository_errors or repository_dependency_errors or tool_dependency_errors: folder_id += 1 - installation_error_base_folder = Folder( id=folder_id, - key='installation_errors', - label='Installation errors', + installation_error_base_folder = Folder( id=folder_id, + key='installation_errors', + label='Installation errors', parent=test_results_folder ) if current_repository_errors: folder_id += 1 - subfolder = Folder( id=folder_id, - key='current_repository_errors', - label='This repository', + subfolder = Folder( id=folder_id, + key='current_repository_errors', + label='This repository', parent=installation_error_base_folder ) repository_error_id = 0 for repository_error_dict in current_repository_errors: @@ -1135,9 +1135,9 @@ def build_tool_test_results_folder( trans, folder_id, tool_test_results_dict, la installation_error_base_folder.folders.append( subfolder ) if repository_dependency_errors: folder_id += 1 - subfolder = Folder( id=folder_id, - key='repository_dependency_errors', - label='Repository dependencies', + subfolder = Folder( id=folder_id, + key='repository_dependency_errors', + label='Repository dependencies', parent=installation_error_base_folder ) repository_error_id = 0 for repository_error_dict in repository_dependency_errors: @@ -1152,9 +1152,9 @@ def build_tool_test_results_folder( trans, folder_id, tool_test_results_dict, la installation_error_base_folder.folders.append( subfolder ) if tool_dependency_errors: folder_id += 1 - subfolder = Folder( id=folder_id, - key='tool_dependency_errors', - label='Tool dependencies', + subfolder = Folder( id=folder_id, + key='tool_dependency_errors', + label='Tool dependencies', parent=installation_error_base_folder ) tool_dependency_error_id = 0 for tool_dependency_error_dict in tool_dependency_errors: @@ -1421,4 +1421,3 @@ def prune_repository_dependencies( folder ): for sub_folder in folder.folders: return prune_repository_dependencies( sub_folder ) return folder - \ No newline at end of file diff --git a/lib/tool_shed/util/data_manager_util.py b/lib/tool_shed/util/data_manager_util.py index a51406a98aa..b4b63635e7a 100644 --- a/lib/tool_shed/util/data_manager_util.py +++ b/lib/tool_shed/util/data_manager_util.py @@ -6,7 +6,7 @@ import tool_shed.util.shed_util_common as suc log = logging.getLogger( __name__ ) def data_manager_config_elems_to_xml_file( app, config_elems, config_filename ):#, shed_tool_conf_filename ): - # Persist the current in-memory list of config_elems to a file named by the value of config_filename. + # Persist the current in-memory list of config_elems to a file named by the value of config_filename. fh = open( config_filename, 'wb' ) fh.write( '\n\n' )#% ( shed_tool_conf_filename )) for elem in config_elems: @@ -95,7 +95,7 @@ def remove_from_data_manager( app, repository ): if metadata_dict and 'data_manager' in metadata_dict: shed_data_manager_conf_filename = app.config.shed_data_manager_config_file tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename ) - if tree: + if tree: root = tree.getroot() assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename ) guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ] diff --git a/lib/tool_shed/util/datatype_util.py b/lib/tool_shed/util/datatype_util.py index 9b2c9924d5c..b56fdbbb39b 100644 --- a/lib/tool_shed/util/datatype_util.py +++ b/lib/tool_shed/util/datatype_util.py @@ -58,7 +58,7 @@ def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_ for relative_path_to_datatype_file_name in datatype_class_modules: datatype_file_name_path, datatype_file_name = os.path.split( relative_path_to_datatype_file_name ) for elem in registration.findall( 'datatype' ): - # Handle 'type' attribute which should be something like one of the following: + # Handle 'type' attribute which should be something like one of the following: # type="gmap:GmapDB" # type="galaxy.datatypes.gmap:GmapDB" dtype = elem.get( 'type', None ) diff --git a/lib/tool_shed/util/export_util.py b/lib/tool_shed/util/export_util.py index 9bb41d59aab..3c1fee0e417 100644 --- a/lib/tool_shed/util/export_util.py +++ b/lib/tool_shed/util/export_util.py @@ -245,7 +245,7 @@ def get_repository_attributes_and_sub_elements( repository, archive_name ): attributes[ 'username' ] = str( repository.user.username ) # Don't coerce description or long description from unicode to string because the fields are free text. sub_elements[ 'description' ] = repository.description - sub_elements[ 'long_description' ] = repository.long_description + sub_elements[ 'long_description' ] = repository.long_description sub_elements[ 'archive' ] = archive_name # Keep track of Category associations. categories = [] diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py index e79f8bc5039..5809970cd3b 100644 --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -91,7 +91,7 @@ def compare_changeset_revisions( trans, ancestor_changeset_revision, ancestor_me current_readme_files = current_metadata_dict.get( 'readme_files', [] ) current_repository_dependencies_dict = current_metadata_dict.get( 'repository_dependencies', {} ) current_repository_dependencies = current_repository_dependencies_dict.get( 'repository_dependencies', [] ) - current_tool_dependencies = current_metadata_dict.get( 'tool_dependencies', {} ) + current_tool_dependencies = current_metadata_dict.get( 'tool_dependencies', {} ) current_workflows = current_metadata_dict.get( 'workflows', [] ) current_data_manager = current_metadata_dict.get( 'data_manager', {} ) # Handle case where no metadata exists for either changeset. @@ -216,7 +216,7 @@ def compare_repository_dependencies( trans, ancestor_repository_dependencies, cu break if not found_in_current: # In some cases, the only difference between a dependency definition in the lists is the changeset_revision value. We'll - # check to see if this is the case, and if the defined dependency is a repository that has metadata set only on it's tip. + # check to see if this is the case, and if the defined dependency is a repository that has metadata set only on it's tip. if not different_revision_defines_tip_only_repository_dependency( trans, ancestor_tup, current_repository_dependencies ): return NOT_EQUAL_AND_NOT_SUBSET return SUBSET @@ -383,8 +383,8 @@ def generate_data_manager_metadata( app, repository, repo_dir, data_manager_conf data_managers = {} invalid_data_managers = [] data_manager_metadata = { 'config_filename': rel_data_manager_config_filename, - 'data_managers': data_managers, - 'invalid_data_managers': invalid_data_managers, + 'data_managers': data_managers, + 'invalid_data_managers': invalid_data_managers, 'error_messages': [] } metadata_dict[ 'data_manager' ] = data_manager_metadata tree, error_message = xml_util.parse_xml( data_manager_config_filename ) @@ -445,12 +445,12 @@ def generate_data_manager_metadata( app, repository, repo_dir, data_manager_conf invalid_data_managers.append( { 'index': i, 'error_message': 'Unable to determine tools metadata' } ) continue - data_managers[ data_manager_id ] = { 'id': data_manager_id, - 'name': data_manager_name, - 'guid': guid, - 'version': version, - 'tool_config_file': data_manager_metadata_tool_file, - 'data_tables': data_tables, + data_managers[ data_manager_id ] = { 'id': data_manager_id, + 'name': data_manager_name, + 'guid': guid, + 'version': version, + 'tool_config_file': data_manager_metadata_tool_file, + 'data_tables': data_tables, 'tool_guid': tool[ 'guid' ] } log.debug( 'Loaded Data Manager tool_files: %s' % ( tool_file ) ) return metadata_dict @@ -563,14 +563,14 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio the repository will have been cloned to a temporary location and updated to a specified changeset revision to access that changeset revision's disk files, so the value of repository_files_dir will not always be repository.repo_path( app ) (it could be an absolute path to a temporary directory containing a clone). If it is an absolute path, the value of relative_install_dir must contain repository.repo_path( app ). - + The value of persist will be True when the installed repository contains a valid tool_data_table_conf.xml.sample file, in which case the entries should ultimately be persisted to the file referred to by app.config.shed_tool_data_table_config. """ if shed_config_dict is None: shed_config_dict = {} if updating_installed_repository: - # Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which + # Keep the original tool shed repository metadata if setting metadata on a repository installed into a local Galaxy instance for which # we have pulled updates. original_repository_metadata = repository.metadata else: @@ -683,8 +683,8 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio work_dir, shed_config_dict, resetting_all_metadata_on_repository ) - - + + metadata_dict = generate_tool_metadata( relative_path_to_tool_config, tool, repository_clone_url, metadata_dict ) else: for tup in invalid_files_and_errors_tups: @@ -706,7 +706,7 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio suc.get_config_from_disk( suc.REPOSITORY_DATA_MANAGER_CONFIG_FILENAME, files_dir ), metadata_dict, shed_config_dict=shed_config_dict ) - + if readme_files: metadata_dict[ 'readme_files' ] = readme_files # This step must be done after metadata for tools has been defined. @@ -1157,14 +1157,14 @@ def handle_repository_elem( app, repository_elem ): log.debug( error_message ) is_valid = False return repository_dependency_tup, is_valid, error_message - else: + else: # We're in the tool shed. if suc.tool_shed_is_this_tool_shed( toolshed ): try: user = sa_session.query( app.model.User ) \ .filter( app.model.User.table.c.username == owner ) \ .one() - except Exception, e: + except Exception, e: error_message = "Ignoring repository dependency definition for tool shed %s, name %s, owner %s, changeset revision %s "% \ ( toolshed, name, owner, changeset_revision ) error_message += "because the owner is invalid. " @@ -1743,12 +1743,12 @@ def set_add_to_tool_panel_attribute_for_tool( tool, guid, datatypes ): if datatypes: for datatype_dict in datatypes: converters = datatype_dict.get( 'converters', None ) - # [{"converters": - # [{"target_datatype": "gff", - # "tool_config": "bed_to_gff_converter.xml", - # "guid": "localhost:9009/repos/test/bed_to_gff_converter/CONVERTER_bed_to_gff_0/2.0.0"}], - # "display_in_upload": "true", - # "dtype": "galaxy.datatypes.interval:Bed", + # [{"converters": + # [{"target_datatype": "gff", + # "tool_config": "bed_to_gff_converter.xml", + # "guid": "localhost:9009/repos/test/bed_to_gff_converter/CONVERTER_bed_to_gff_0/2.0.0"}], + # "display_in_upload": "true", + # "dtype": "galaxy.datatypes.interval:Bed", # "extension": "bed"}] if converters: for converter_dict in converters: @@ -1888,12 +1888,12 @@ def tool_dependency_is_orphan( type, name, version, tools ): def update_existing_tool_dependency( app, repository, original_dependency_dict, new_dependencies_dict ): """ - Update an exsiting tool dependency whose definition was updated in a change set pulled by a Galaxy administrator when getting updates + Update an exsiting tool dependency whose definition was updated in a change set pulled by a Galaxy administrator when getting updates to an installed tool shed repository. The original_dependency_dict is a single tool dependency definition, an example of which is:: - {"name": "bwa", - "readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ", - "type": "package", + {"name": "bwa", + "readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ", + "type": "package", "version": "0.6.2"} The new_dependencies_dict is the dictionary generated by the metadata_util.generate_tool_dependency_metadata method. diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py index bd58a4942c8..0564d966285 100644 --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -28,7 +28,7 @@ def build_readme_files_dict( metadata, tool_path=None ): def get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ): """ - Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section + Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section selection page. """ name = repo_info_dict.keys()[ 0 ] diff --git a/lib/tool_shed/util/repository_dependency_util.py b/lib/tool_shed/util/repository_dependency_util.py index 0c9ff3293e2..a1811a8ee8e 100644 --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -64,7 +64,7 @@ def build_repository_dependency_relationships( trans, repo_info_dicts, tool_shed break if required_repository is None: # The required repository is not in the received list so look in the database. - required_repository = suc.get_or_create_tool_shed_repository( trans, rd_toolshed, rd_name, rd_owner, rd_changeset_revision ) + required_repository = suc.get_or_create_tool_shed_repository( trans, rd_toolshed, rd_name, rd_owner, rd_changeset_revision ) # Ensure there is a repository_dependency relationship between dependent_repository and required_repository. rrda = None for rd in dependent_repository.repository_dependencies: @@ -214,7 +214,7 @@ def create_repository_dependency_objects( trans, tool_path, tool_shed_url, repo_ tool_panel_section_key, tool_section = tool_util.handle_tool_panel_section( trans, tool_panel_section=tool_panel_section, new_tool_panel_section=new_tool_panel_section ) - + else: # We're installing a new tool shed repository that does not yet have a database record. tool_panel_section_key, tool_section = tool_util.handle_tool_panel_section( trans, @@ -266,7 +266,7 @@ def get_key_for_repository_changeset_revision( toolshed_base_url, repository, re repository_name=repository.name, repository_owner=repository.user.username, changeset_revision=repository_metadata.changeset_revision, - prior_installation_required=prior_installation_required ) + prior_installation_required=prior_installation_required ) return key def get_prior_installation_required_for_key( toolshed_base_url, repository, repository_metadata, all_repository_dependencies ): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 7f6c7c648e8..044976128cc 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -30,7 +30,7 @@ from mercurial import ui eggs.require( 'markupsafe' ) import markupsafe - + log = logging.getLogger( __name__ ) CHUNK_SIZE = 2**20 # 1Mb @@ -194,7 +194,7 @@ def clean_tool_shed_url( tool_shed_url ): return tool_shed_url.split( ':' )[ 0 ] return tool_shed_url.rstrip( '/' ) -def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): +def clone_repository( repository_clone_url, repository_file_dir, ctx_rev ): """Clone the repository up to the specified changeset_revision. No subsequent revisions will be present in the cloned repository.""" try: commands.clone( get_configured_ui(), @@ -248,7 +248,7 @@ def create_or_update_tool_shed_repository( app, name, description, installed_cha # was later uninstalled, this value should be received as the value of that change set to which the repository had been updated just prior # to it being uninstalled. current_changeset_revision = installed_changeset_revision - sa_session = app.model.context.current + sa_session = app.model.context.current tool_shed = get_tool_shed_from_clone_url( repository_clone_url ) if not owner: owner = get_repository_owner_from_clone_url( repository_clone_url ) @@ -322,7 +322,7 @@ def generate_repository_info_elem( tool_shed, repository_name, changeset_revisio elem = XmlET.Element( 'tool_shed_repository' ) else: elem = XmlET.SubElement( parent_elem, 'tool_shed_repository' ) - + tool_shed_elem = XmlET.SubElement( elem, 'tool_shed' ) tool_shed_elem.text = tool_shed repository_name_elem = XmlET.SubElement( elem, 'repository_name' ) @@ -337,10 +337,10 @@ def generate_repository_info_elem( tool_shed, repository_name, changeset_revisio new_elem = XmlET.SubElement( elem, key ) new_elem.text = value return elem - + def generate_repository_info_elem_from_repository( tool_shed_repository, parent_elem=None, **kwd ): return generate_repository_info_elem( tool_shed_repository.tool_shed, tool_shed_repository.name, tool_shed_repository.installed_changeset_revision, tool_shed_repository.owner, parent_elem=parent_elem, **kwd ) - + def generate_sharable_link_for_repository_in_tool_shed( trans, repository, changeset_revision=None ): """Generate the URL for sharing a repository that is in the tool shed.""" @@ -376,7 +376,7 @@ def generate_tool_elem( tool_shed, repository_name, changeset_revision, owner, t def generate_tool_guid( repository_clone_url, tool ): """ Generate a guid for the installed tool. It is critical that this guid matches the guid for - the tool in the Galaxy tool shed from which it is being installed. The form of the guid is + the tool in the Galaxy tool shed from which it is being installed. The form of the guid is /repos//// """ tmp_url = clean_repository_clone_url( repository_clone_url ) @@ -399,7 +399,7 @@ def generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository ): tool_config = tool_dict[ 'tool_config' ] file_name = strip_path( tool_config ) guids_and_configs[ guid ] = file_name - # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. + # Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict. tree, error_message = xml_util.parse_xml( shed_tool_conf ) if tree is None: return tool_panel_dict @@ -517,7 +517,7 @@ def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ): Send a request to the tool shed to retrieve the ctx_rev for a repository defined by the combination of a name, owner and changeset revision. """ - url = url_join( tool_shed_url, + url = url_join( tool_shed_url, 'repository/get_ctx_rev?name=%s&owner=%s&changeset_revision=%s' % ( name, owner, changeset_revision ) ) ctx_rev = common_util.tool_shed_get( app, tool_shed_url, url ) return ctx_rev @@ -1349,7 +1349,7 @@ def repository_was_previously_installed( trans, tool_shed_url, repository_name, def reset_previously_installed_repository( trans, repository ): """ - Reset the atrributes of a tool_shed_repository that was previsouly installed. The repository will be in some state other than with a + Reset the atrributes of a tool_shed_repository that was previsouly installed. The repository will be in some state other than with a status of INSTALLED, so all atributes will be set to the default NEW state. This will enable the repository to be freshly installed. """ repository.deleted = False @@ -1359,7 +1359,7 @@ def reset_previously_installed_repository( trans, repository ): repository.error_message = None trans.sa_session.add( repository ) trans.sa_session.flush() - + def reversed_lower_upper_bounded_changelog( repo, excluded_lower_bounds_changeset_revision, included_upper_bounds_changeset_revision ): """ Return a reversed list of changesets in the repository changelog after the excluded_lower_bounds_changeset_revision, but up to and diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index 8449c27992f..fa2fd5c7063 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -61,7 +61,7 @@ def add_to_tool_panel( app, repository_name, repository_clone_url, changeset_rev if new_install: # Add the new elements to the shed_tool_conf file on disk. add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ) - # Use the new elements to add entries to the + # Use the new elements to add entries to the config_elems = shed_tool_conf_dict[ 'config_elems' ] for config_elem in elem_list: # Add the new elements to the in-memory list of config_elems. @@ -136,7 +136,7 @@ def can_use_tool_config_disk_file( trans, repository, repo, file_path, changeset def check_tool_input_params( app, repo_dir, tool_config_name, tool, sample_files ): """ - Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make + Check all of the tool's input parameters, looking for any that are dynamically generated using external data files to make sure the files exist. """ invalid_files_and_errors_tups = [] @@ -597,7 +597,7 @@ def handle_sample_files_and_load_tool_from_tmp_config( trans, repo, repository_i tool = None message = '' ctx = suc.get_changectx_for_changeset( repo, changeset_revision ) - # We're not currently doing anything with the returned list of deleted_sample_files here. It is intended to help handle sample files that are in + # We're not currently doing anything with the returned list of deleted_sample_files here. It is intended to help handle sample files that are in # the manifest, but have been deleted from disk. sample_files, deleted_sample_files = get_list_of_copied_sample_files( repo, ctx, dir=work_dir ) if sample_files: @@ -758,9 +758,9 @@ def install_tool_data_tables( app, tool_shed_repository, tool_index_sample_files if elems: os.unlink( tool_data_table_conf_filename ) #remove old data_table app.tool_data_tables.to_xml_file( tool_data_table_conf_filename, elems ) #persist new data_table content - + return tool_data_table_conf_filename, elems - + def is_column_based( fname, sep='\t', skip=0, is_multi_byte=False ): """See if the file is column based with respect to a separator.""" diff --git a/lib/tool_shed/util/workflow_util.py b/lib/tool_shed/util/workflow_util.py index 3e3dfbe7d43..10b167b56ea 100644 --- a/lib/tool_shed/util/workflow_util.py +++ b/lib/tool_shed/util/workflow_util.py @@ -47,9 +47,9 @@ class RepoInputDataModule( InputDataModule ): class RepoToolModule( ToolModule ): - + type = "tool" - + def __init__( self, trans, repository_id, changeset_revision, tools_metadata, tool_id ): self.trans = trans self.tools_metadata = tools_metadata @@ -267,7 +267,7 @@ def generate_workflow_image( trans, workflow_name, repository_metadata_id=None, if trans.webapp.name == 'tool_shed' and tool_unavailable: fill = "#EBBCB2" else: - fill = "#EBD9B2" + fill = "#EBD9B2" boxes.append( svgfig.Rect( x - margin, y, x + width - margin, y + 30, fill=fill ).SVG() ) box_height = ( len( step_dict[ 'data_inputs' ] ) + len( step_dict[ 'data_outputs' ] ) ) * line_px + margin # Draw separator line. @@ -351,7 +351,7 @@ def get_workflow_data_outputs( step, module, steps ): if not found: # We're at the last step of the workflow. data_outputs_dict[ 'name' ] = 'output' - data_outputs.append( data_outputs_dict ) + data_outputs.append( data_outputs_dict ) return data_outputs return module.get_data_outputs() diff --git a/lib/tool_shed/util/xml_util.py b/lib/tool_shed/util/xml_util.py index 4ee96fb424f..ee6c62272d2 100644 --- a/lib/tool_shed/util/xml_util.py +++ b/lib/tool_shed/util/xml_util.py @@ -16,7 +16,7 @@ class Py26CommentedTreeBuilder ( XmlET.XMLTreeBuilder ): def __init__ ( self, html=0, target=None ): XmlET.XMLTreeBuilder.__init__( self, html, target ) self._parser.CommentHandler = self.handle_comment - + def handle_comment ( self, data ): self._target.start( XmlET.Comment, {} ) self._target.data( data ) @@ -25,7 +25,7 @@ class Py26CommentedTreeBuilder ( XmlET.XMLTreeBuilder ): class Py27CommentedTreeBuilder ( XmlET.TreeBuilder ): # Python 2.7 uses ElementTree 1.3.x. - + def comment( self, data ): self.start( XmlET.Comment, {} ) self.data( data )