From b33b330a81360e531d2dcf079ccd3f398c420896 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 10 Nov 2012 14:26:29 -0600 Subject: [PATCH 001/318] Refactor setting up valid extensions in DataToolParameter to eliminate code duplication, extra computation (multiple loading of same datatypes in unit tests for instance), and reduce code nesting. --- lib/galaxy/tools/parameters/basic.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index d6d2352ca8c..3fbc8a57623 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1371,24 +1371,25 @@ class DataToolParameter( ToolParameter ): # Add metadata validator if not string_as_bool( elem.get( 'no_validation', False ) ): self.validators.append( validation.MetadataValidator() ) + # Find datatypes_registry + if tool is None: + if trans: + # Must account for "Input Dataset" types, which while not a tool still need access to the real registry. + # A handle to the transaction (and thus app) will be given by the module. + datatypes_registry = trans.app.datatypes_registry + else: + #This occurs for things such as unit tests + import galaxy.datatypes.registry + datatypes_registry = galaxy.datatypes.registry.Registry() + datatypes_registry.load_datatypes() + else: + datatypes_registry = tool.app.datatypes_registry # Build tuple of classes for supported data formats formats = [] self.extensions = elem.get( 'format', 'data' ).split( "," ) for extension in self.extensions: extension = extension.strip() - if tool is None: - if trans: - # Must account for "Input Dataset" types, which while not a tool still need access to the real registry. - # A handle to the transaction (and thus app) will be given by the module. - formats.append( trans.app.datatypes_registry.get_datatype_by_extension( extension.lower() ).__class__ ) - else: - #This occurs for things such as unit tests - import galaxy.datatypes.registry - datatypes_registry = galaxy.datatypes.registry.Registry() - datatypes_registry.load_datatypes() - formats.append( datatypes_registry.get_datatype_by_extension( extension.lower() ).__class__ ) - else: - formats.append( tool.app.datatypes_registry.get_datatype_by_extension( extension.lower() ).__class__ ) + formats.append( datatypes_registry.get_datatype_by_extension( extension.lower() ).__class__ ) self.formats = tuple( formats ) self.multiple = string_as_bool( elem.get( 'multiple', False ) ) # TODO: Enhance dynamic options for DataToolParameters. Currently, From 92e833d3ede37504ca4f97a48faf6a410f8ea9ba Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 15 Nov 2012 08:11:02 -0600 Subject: [PATCH 002/318] Centeralize logic for checking if a given galaxy datatype is any of the datatypes in a given iterator (tuple, list) (used when building up tool input options and evaluating tools, checking conversions, etc...). I think this is a solid (though slight) design improvement in its own right, but it will also ease the eventually merging of composite multifiles (https://bitbucket.org/galaxyp/galaxy-central-homogeneous-composite-datatypes) into galaxy-central or at least limit the divergence between galaxy-central and that fork long term. --- lib/galaxy/datatypes/data.py | 8 +++++++- lib/galaxy/datatypes/registry.py | 2 +- lib/galaxy/tools/actions/__init__.py | 10 ++++++---- lib/galaxy/tools/parameters/basic.py | 14 +++++++------- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index e641877f3b8..f411eb10a6c 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -529,7 +529,13 @@ class Data( object ): def has_resolution(self): return False - + def matches_any( self, target_datatypes ): + """ + Check if this datatype is of any of the target_datatypes or is + a subtype thereof. + """ + datatype_classes = tuple( [ datatype.__class__ for datatype in target_datatypes ] ) + return isinstance( self, datatype_classes ) def merge( split_files, output_file): """ diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 0a4ad3c17d5..24e234ecc9c 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -586,7 +586,7 @@ class Registry( object ): def find_conversion_destination_for_dataset_by_extensions( self, dataset, accepted_formats, converter_safe = True ): """Returns ( target_ext, existing converted dataset )""" for convert_ext in self.get_converters_by_datatype( dataset.ext ): - if isinstance( self.get_datatype_by_extension( convert_ext ), accepted_formats ): + if self.get_datatype_by_extension( convert_ext ).matches_any( accepted_formats ): converted_dataset = dataset.get_converted_files_by_type( convert_ext ) if converted_dataset: ret_data = converted_dataset diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 1b1a4757912..e99f19109a8 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -36,7 +36,7 @@ class DefaultToolAction( object ): def process_dataset( data, formats = None ): if formats is None: formats = input.formats - if data and not isinstance( data.datatype, formats ): + if data and not data.datatype.matches_any( formats ): # Need to refresh in case this conversion just took place, i.e. input above in tool performed the same conversion trans.sa_session.refresh( data ) target_ext, converted_dataset = data.find_conversion_destination( formats ) @@ -66,7 +66,7 @@ class DefaultToolAction( object ): conversions = [] for conversion_name, conversion_extensions, conversion_datatypes in input.conversions: new_data = process_dataset( input_datasets[ prefix + input.name + str( i + 1 ) ], conversion_datatypes ) - if not new_data or isinstance( new_data.datatype, conversion_datatypes ): + if not new_data or new_data.datatype.matches_any( conversion_datatypes ): input_datasets[ prefix + conversion_name + str( i + 1 ) ] = new_data conversions.append( ( conversion_name, new_data ) ) else: @@ -86,7 +86,7 @@ class DefaultToolAction( object ): conversions = [] for conversion_name, conversion_extensions, conversion_datatypes in input.conversions: new_data = process_dataset( input_datasets[ prefix + input.name ], conversion_datatypes ) - if not new_data or isinstance( new_data.datatype, conversion_datatypes ): + if not new_data or new_data.datatype.matches_any( conversion_datatypes ): input_datasets[ prefix + conversion_name ] = new_data conversions.append( ( conversion_name, new_data ) ) else: @@ -261,7 +261,9 @@ class DefaultToolAction( object ): ext = input_ext if output.format_source is not None and output.format_source in inp_data: try: - ext = inp_data[output.format_source].ext + input_dataset = inp_data[output.format_source] + input_extension = input_dataset.ext + ext = input_extension except Exception, e: pass diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 3fbc8a57623..244c14baf14 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -785,7 +785,7 @@ class SelectToolParameter( ToolParameter ): if isinstance( dep_value, RuntimeValue ): return True #dataset not ready yet - if hasattr( self, 'ref_input' ) and isinstance( dep_value, self.tool.app.model.HistoryDatasetAssociation ) and ( dep_value.is_pending or not isinstance( dep_value.datatype, self.ref_input.formats ) ): + if hasattr( self, 'ref_input' ) and isinstance( dep_value, self.tool.app.model.HistoryDatasetAssociation ) and ( dep_value.is_pending or not dep_value.datatype.matches_any( self.ref_input.formats ) ): return True # Dynamic, but all dependenceis are known and have values return False @@ -1037,7 +1037,7 @@ class ColumnListParameter( SelectToolParameter ): if not dataset.metadata.columns: # Only allow late validation if the dataset is not yet ready # (since we have reason to expect the metadata to be ready eventually) - if dataset.is_pending or not isinstance( dataset.datatype, self.ref_input.formats ): + if dataset.is_pending or not dataset.datatype.matches_any( self.ref_input.formats ): return True # No late validation return False @@ -1389,8 +1389,8 @@ class DataToolParameter( ToolParameter ): self.extensions = elem.get( 'format', 'data' ).split( "," ) for extension in self.extensions: extension = extension.strip() - formats.append( datatypes_registry.get_datatype_by_extension( extension.lower() ).__class__ ) - self.formats = tuple( formats ) + formats.append( datatypes_registry.get_datatype_by_extension( extension.lower() ) ) + self.formats = formats self.multiple = string_as_bool( elem.get( 'multiple', False ) ) # TODO: Enhance dynamic options for DataToolParameters. Currently, # only the special case key='build' of type='data_meta' is @@ -1414,7 +1414,7 @@ class DataToolParameter( ToolParameter ): conv_extensions = conv_elem.get( "type" ) #target datatype extension # FIXME: conv_extensions should be able to be an ordered list assert None not in [ name, type ], 'A name (%s) and type (%s) are required for explicit conversion' % ( name, type ) - conv_types = tool.app.datatypes_registry.get_datatype_by_extension( conv_extensions.lower() ).__class__ + conv_types = tool.app.datatypes_registry.get_datatype_by_extension( conv_extensions.lower() ) self.conversions.append( ( name, conv_extensions, conv_types ) ) def get_html_field( self, trans=None, value=None, other_values={} ): @@ -1452,7 +1452,7 @@ class DataToolParameter( ToolParameter ): continue if self.options and self._options_filter_attribute( hda ) != filter_value: continue - if isinstance( hda.datatype, self.formats): + if hda.datatype.matches_any( self.formats ): selected = ( value and ( hda in value ) ) if hda.visible: hidden_text = "" @@ -1519,7 +1519,7 @@ class DataToolParameter( ToolParameter ): for i, data in enumerate( datasets ): if data.visible and not data.deleted and data.state not in [data.states.ERROR, data.states.DISCARDED]: is_valid = False - if isinstance( data.datatype, self.formats ): + if data.datatype.matches_any( self.formats ): is_valid = True else: target_ext, converted_dataset = data.find_conversion_destination( self.formats ) From 063e0a0080ca4e025ac24a26c28f8dc2be869d43 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 15 Nov 2012 08:23:15 -0600 Subject: [PATCH 003/318] Small refactoring of logic used to build up list of valid formats for an input data parameter. --- lib/galaxy/tools/parameters/basic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 244c14baf14..5b4556e4265 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1387,9 +1387,9 @@ class DataToolParameter( ToolParameter ): # Build tuple of classes for supported data formats formats = [] self.extensions = elem.get( 'format', 'data' ).split( "," ) - for extension in self.extensions: - extension = extension.strip() - formats.append( datatypes_registry.get_datatype_by_extension( extension.lower() ) ) + normalized_extensions = [extension.strip().lower() for extension in self.extensions] + for extension in normalized_extensions: + formats.append( datatypes_registry.get_datatype_by_extension( extension ) ) self.formats = formats self.multiple = string_as_bool( elem.get( 'multiple', False ) ) # TODO: Enhance dynamic options for DataToolParameters. Currently, From 1b88fb40f994f225f8285e37eb118dbadf2373fa Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 3 Dec 2012 23:04:36 -0600 Subject: [PATCH 004/318] Leave process_dataset immediately if input data is `None` in collect_input_datasets of DefaultToolAction, simplifies code and eliminates some unneeded computation. --- lib/galaxy/tools/actions/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index e99f19109a8..a94b6ddf932 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -34,9 +34,11 @@ class DefaultToolAction( object ): input_datasets = dict() def visitor( prefix, input, value, parent = None ): def process_dataset( data, formats = None ): + if not data: + return data if formats is None: formats = input.formats - if data and not data.datatype.matches_any( formats ): + if not data.datatype.matches_any( formats ): # Need to refresh in case this conversion just took place, i.e. input above in tool performed the same conversion trans.sa_session.refresh( data ) target_ext, converted_dataset = data.find_conversion_destination( formats ) @@ -54,7 +56,7 @@ class DefaultToolAction( object ): trans.sa_session.flush() data = new_data current_user_roles = trans.get_current_user_roles() - if data and not trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ): + if not trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ): raise "User does not have permission to use a dataset (%s) provided for input." % data.id return data if isinstance( input, DataToolParameter ): From f98cf656a286512510a2c7ce13f8d0d0745595f9 Mon Sep 17 00:00:00 2001 From: Lance Parsons Date: Mon, 10 Dec 2012 16:23:11 -0500 Subject: [PATCH 005/318] Fix for Postgres versions < 9 (see https://trello.com/c/Z0EynAV2) --- scripts/set_user_disk_usage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/set_user_disk_usage.py b/scripts/set_user_disk_usage.py index 213c9086a73..153d8ba1c25 100755 --- a/scripts/set_user_disk_usage.py +++ b/scripts/set_user_disk_usage.py @@ -52,7 +52,7 @@ def pgcalc( sa_session, id ): sql = """ UPDATE galaxy_user SET disk_usage = (SELECT COALESCE(SUM(total_size), 0) - FROM ( SELECT d.total_size + FROM ( SELECT DISTINCT ON (d.id) d.total_size, d.id FROM history_dataset_association hda JOIN history h ON h.id = hda.history_id JOIN dataset d ON hda.dataset_id = d.id @@ -62,7 +62,7 @@ def pgcalc( sa_session, id ): AND d.purged = false AND d.id NOT IN (SELECT dataset_id FROM library_dataset_dataset_association) - GROUP BY d.id) sizes) + ) sizes) WHERE id = :id RETURNING disk_usage; """ From 8607681e477c022c7c19872ee1acfc4ca060921b Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 10 Apr 2013 12:56:49 -0400 Subject: [PATCH 006/318] Branch close from search API. From 9019b1c68bb89131ed14fa79827b621ed29c6c3b Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 24 Jun 2013 14:15:46 +0200 Subject: [PATCH 007/318] Use sam_fa_indexes tool data table instead of searching the dbkey in sam_fa_indices.loc Also remove some unused imports. --- tools/samtools/sam_pileup.py | 25 ++++---------------- tools/samtools/sam_pileup.xml | 15 ++++++++---- tools/samtools/sam_to_bam.py | 36 ++++------------------------- tools/samtools/sam_to_bam.xml | 21 ++++++++++------- tools/samtools/samtools_mpileup.xml | 18 +++++++-------- 5 files changed, 41 insertions(+), 74 deletions(-) diff --git a/tools/samtools/sam_pileup.py b/tools/samtools/sam_pileup.py index 50168fc2fe6..b236c95ed84 100644 --- a/tools/samtools/sam_pileup.py +++ b/tools/samtools/sam_pileup.py @@ -8,8 +8,7 @@ usage: %prog [options] -o, --output1=o: Output pileup -R, --ref=R: Reference file type -n, --ownFile=n: User-supplied fasta reference file - -d, --dbkey=d: dbkey of user-supplied file - -x, --indexDir=x: Index directory + -g, --index=g: Path of the indexed reference genome -b, --bamIndex=b: BAM index file -s, --lastCol=s: Print the mapping quality as the last column -i, --indels=i: Only output lines containing indels @@ -31,24 +30,9 @@ def stop_err( msg ): sys.stderr.write( '%s\n' % msg ) sys.exit() -def check_seq_file( dbkey, GALAXY_DATA_INDEX_DIR ): - seqFile = '%s/sam_fa_indices.loc' % GALAXY_DATA_INDEX_DIR - seqPath = '' - for line in open( seqFile ): - line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#' ) and line.startswith( 'index' ): - fields = line.split( '\t' ) - if len( fields ) < 3: - continue - if fields[1] == dbkey: - seqPath = fields[2].strip() - break - return seqPath - def __main__(): #Parse Command Line options, args = doc_optparse.parse( __doc__ ) - seqPath = check_seq_file( options.dbkey, options.indexDir ) # output version # of tool try: tmp = tempfile.NamedTemporaryFile().name @@ -77,7 +61,6 @@ def __main__(): tmpf1 = tempfile.NamedTemporaryFile( dir=tmpDir ) tmpf1_name = tmpf1.name tmpf1.close() - tmpf1fai_name = '%s.fai' % tmpf1_name #link bam and bam index to working directory (can't move because need to leave original) os.symlink( options.input1, tmpf0bam_name ) os.symlink( options.bamIndex, tmpf0bambai_name ) @@ -100,9 +83,9 @@ def __main__(): try: #index reference if necessary and prepare pileup command if options.ref == 'indexed': - if not os.path.exists( "%s.fai" % seqPath ): - raise Exception, "No sequences are available for '%s', request them by reporting this error." % options.dbkey - cmd = cmd % ( opts, seqPath, tmpf0bam_name, options.output1 ) + if not os.path.exists( "%s.fai" % options.index ): + raise Exception, "Indexed genome %s not present, request it by reporting this error." % options.index + cmd = cmd % ( opts, options.index, tmpf0bam_name, options.output1 ) elif options.ref == 'history': os.symlink( options.ownFile, tmpf1_name ) cmdIndex = 'samtools faidx %s' % ( tmpf1_name ) diff --git a/tools/samtools/sam_pileup.xml b/tools/samtools/sam_pileup.xml index e51377f5aac..37897b395cc 100644 --- a/tools/samtools/sam_pileup.xml +++ b/tools/samtools/sam_pileup.xml @@ -1,4 +1,4 @@ - + from BAM dataset samtools @@ -11,10 +11,8 @@ #if $refOrHistory.reference == "history": --ownFile=$refOrHistory.ownFile #else: - --ownFile="None" + --index=${refOrHistory.index.fields.path} #end if - --dbkey=${input1.metadata.dbkey} - --indexDir=${GALAXY_DATA_INDEX_DIR} --bamIndex=${input1.metadata.bam_index} --lastCol=$lastCol --indels=$indels @@ -41,7 +39,13 @@ - + + + + + + + @@ -100,6 +104,7 @@ --> + diff --git a/tools/samtools/sam_to_bam.py b/tools/samtools/sam_to_bam.py index a13e982e774..f209a7e41c0 100644 --- a/tools/samtools/sam_to_bam.py +++ b/tools/samtools/sam_to_bam.py @@ -3,43 +3,24 @@ Converts SAM data to sorted BAM data. usage: sam_to_bam.py [options] --input1: SAM file to be converted - --dbkey: dbkey value + --index: path of the indexed reference genome --ref_file: Reference file if choosing from history --output1: output dataset in bam format - --index_dir: GALAXY_DATA_INDEX_DIR """ -import optparse, os, sys, subprocess, tempfile, shutil, gzip -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -from bx.cookbook import doc_optparse -from galaxy import util +import optparse, os, sys, subprocess, tempfile, shutil def stop_err( msg ): sys.stderr.write( '%s\n' % msg ) sys.exit() -def check_seq_file( dbkey, cached_seqs_pointer_file ): - seq_path = '' - for line in open( cached_seqs_pointer_file ): - line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#' ) and line.startswith( 'index' ): - fields = line.split( '\t' ) - if len( fields ) < 3: - continue - if fields[1] == dbkey: - seq_path = fields[2].strip() - break - return seq_path - def __main__(): #Parse Command Line parser = optparse.OptionParser() parser.add_option( '', '--input1', dest='input1', help='The input SAM dataset' ) - parser.add_option( '', '--dbkey', dest='dbkey', help='The build of the reference dataset' ) + parser.add_option( '', '--index', dest='index', help='The path of the indexed reference genome' ) parser.add_option( '', '--ref_file', dest='ref_file', help='The reference dataset from the history' ) parser.add_option( '', '--output1', dest='output1', help='The output BAM dataset' ) - parser.add_option( '', '--index_dir', dest='index_dir', help='GALAXY_DATA_INDEX_DIR' ) ( options, args ) = parser.parse_args() # output version # of tool @@ -61,24 +42,17 @@ def __main__(): except: sys.stdout.write( 'Could not determine Samtools version\n' ) - cached_seqs_pointer_file = '%s/sam_fa_indices.loc' % options.index_dir - if not os.path.exists( cached_seqs_pointer_file ): - stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file ) - # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa, - # and the equCab2.fa file will contain fasta sequences. - seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file ) tmp_dir = tempfile.mkdtemp() if not options.ref_file or options.ref_file == 'None': # We're using locally cached reference sequences( e.g., /galaxy/data/equCab2/sam_index/equCab2.fa ). # The indexes for /galaxy/data/equCab2/sam_index/equCab2.fa will be contained in # a file named /galaxy/data/equCab2/sam_index/equCab2.fa.fai - fai_index_file_base = seq_path - fai_index_file_path = '%s.fai' % seq_path + fai_index_file_path = '%s.fai' % options.index if not os.path.exists( fai_index_file_path ): #clean up temp files if os.path.exists( tmp_dir ): shutil.rmtree( tmp_dir ) - stop_err( 'No sequences are available for build (%s), request them by reporting this error.' % options.dbkey ) + stop_err( 'Indexed genome %s not present, request it by reporting this error.' % options.index ) else: try: # Create indexes for history reference ( e.g., ~/database/files/000/dataset_1.dat ) using samtools faidx, which will: diff --git a/tools/samtools/sam_to_bam.xml b/tools/samtools/sam_to_bam.xml index 7364bb4fdac..03a0c478421 100644 --- a/tools/samtools/sam_to_bam.xml +++ b/tools/samtools/sam_to_bam.xml @@ -1,4 +1,4 @@ - + converts SAM format to BAM format samtools @@ -7,13 +7,11 @@ sam_to_bam.py --input1=$source.input1 #if $source.index_source == "history": - --dbkey=${ref_file.metadata.dbkey} --ref_file=$source.ref_file #else - --dbkey=${input1.metadata.dbkey} + --index=${source.index.fields.path} #end if --output1=$output1 - --index_dir=${GALAXY_DATA_INDEX_DIR} @@ -22,13 +20,19 @@ - - - + + + + + + + + + - + @@ -76,6 +80,7 @@ --> + diff --git a/tools/samtools/samtools_mpileup.xml b/tools/samtools/samtools_mpileup.xml index a8280630e77..81a7f3dbf2f 100644 --- a/tools/samtools/samtools_mpileup.xml +++ b/tools/samtools/samtools_mpileup.xml @@ -1,4 +1,4 @@ - + SNP and indel caller samtools @@ -59,22 +59,22 @@ - - - - + + + + - + - - - + + + From 6001e2549e580d754b3535346afa6d519c0ef701 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 24 Jun 2013 14:17:42 +0200 Subject: [PATCH 008/318] Use sam_fa_indexes tool data table instead of searching the dbkey in sam_fa_indices.loc also in Cufflinks wrappers --- tools/ngs_rna/cuffcompare_wrapper.py | 30 +++++---------------------- tools/ngs_rna/cuffcompare_wrapper.xml | 17 +++++++++------ tools/ngs_rna/cuffdiff_wrapper.py | 30 +++++---------------------- tools/ngs_rna/cuffdiff_wrapper.xml | 17 +++++++++------ tools/ngs_rna/cufflinks_wrapper.py | 30 +++++---------------------- tools/ngs_rna/cufflinks_wrapper.xml | 25 +++++++++++++--------- tools/ngs_rna/cuffmerge_wrapper.py | 30 +++++---------------------- tools/ngs_rna/cuffmerge_wrapper.xml | 17 +++++++++------ 8 files changed, 68 insertions(+), 128 deletions(-) diff --git a/tools/ngs_rna/cuffcompare_wrapper.py b/tools/ngs_rna/cuffcompare_wrapper.py index dba53ec434e..9d9ad373abd 100644 --- a/tools/ngs_rna/cuffcompare_wrapper.py +++ b/tools/ngs_rna/cuffcompare_wrapper.py @@ -8,20 +8,6 @@ def stop_err( msg ): sys.stderr.write( '%s\n' % msg ) sys.exit() -# Copied from sam_to_bam.py: -def check_seq_file( dbkey, cached_seqs_pointer_file ): - seq_path = '' - for line in open( cached_seqs_pointer_file ): - line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#' ) and line.startswith( 'index' ): - fields = line.split( '\t' ) - if len( fields ) < 3: - continue - if fields[1] == dbkey: - seq_path = fields[2].strip() - break - return seq_path - def __main__(): #Parse Command Line parser = optparse.OptionParser() @@ -30,8 +16,7 @@ def __main__(): parser.add_option( '-s', dest='use_seq_data', action="store_true", help='Causes cuffcompare to look into for fasta files with the underlying genomic sequences (one file per contig) against which your reads were aligned for some optional classification functions. For example, Cufflinks transcripts consisting mostly of lower-case bases are classified as repeats. Note that must contain one fasta file per reference chromosome, and each file must be named after the chromosome, and have a .fa or .fasta extension.') # Wrapper / Galaxy options. - parser.add_option( '', '--dbkey', dest='dbkey', help='The build of the reference dataset' ) - parser.add_option( '', '--index_dir', dest='index_dir', help='GALAXY_DATA_INDEX_DIR' ) + parser.add_option( '', '--index', dest='index', help='The path of the reference genome' ) parser.add_option( '', '--ref_file', dest='ref_file', help='The reference dataset from the history' ) # Outputs. @@ -60,21 +45,16 @@ def __main__(): # Set/link to sequence file. if options.use_seq_data: - if options.ref_file != 'None': + if options.ref_file: # Sequence data from history. # Create symbolic link to ref_file so that index will be created in working directory. seq_path = "ref.fa" os.symlink( options.ref_file, seq_path ) else: # Sequence data from loc file. - cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' ) - if not os.path.exists( cached_seqs_pointer_file ): - stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file ) - # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa, - # and the equCab2.fa file will contain fasta sequences. - seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file ) - if seq_path == '': - stop_err( 'No sequence data found for dbkey %s, so sequence data cannot be used.' % options.dbkey ) + if not os.path.exists( options.index ): + stop_err( 'Reference genome %s not present, request it by reporting this error.' % options.index ) + seq_path = options.index # Build command. diff --git a/tools/ngs_rna/cuffcompare_wrapper.xml b/tools/ngs_rna/cuffcompare_wrapper.xml index 11e0ca75f0d..dc221c5d803 100644 --- a/tools/ngs_rna/cuffcompare_wrapper.xml +++ b/tools/ngs_rna/cuffcompare_wrapper.xml @@ -1,4 +1,4 @@ - + compare assembled transcripts to a reference annotation and track Cufflinks transcripts across multiple experiments @@ -17,14 +17,12 @@ ## Use sequence data? #if $seq_data.use_seq_data == "Yes": - -s + -s #if $seq_data.seq_source.index_source == "history": --ref_file=$seq_data.seq_source.ref_file #else: - --ref_file="None" + --index=${seq_data.seq_source.index.fields.path} #end if - --dbkey=${first_input.metadata.dbkey} - --index_dir=${GALAXY_DATA_INDEX_DIR} #end if ## Outputs. @@ -66,7 +64,14 @@ - + + + + + + + + diff --git a/tools/ngs_rna/cuffdiff_wrapper.py b/tools/ngs_rna/cuffdiff_wrapper.py index 4ba463df1b8..000ac2c09df 100644 --- a/tools/ngs_rna/cuffdiff_wrapper.py +++ b/tools/ngs_rna/cuffdiff_wrapper.py @@ -35,20 +35,6 @@ def stop_err( msg ): sys.stderr.write( "%s\n" % msg ) sys.exit() -# Copied from sam_to_bam.py: -def check_seq_file( dbkey, cached_seqs_pointer_file ): - seq_path = '' - for line in open( cached_seqs_pointer_file ): - line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#' ) and line.startswith( 'index' ): - fields = line.split( '\t' ) - if len( fields ) < 3: - continue - if fields[1] == dbkey: - seq_path = fields[2].strip() - break - return seq_path - def __main__(): #Parse Command Line parser = optparse.OptionParser() @@ -83,8 +69,7 @@ def __main__(): # Bias correction options. parser.add_option( '-b', dest='do_bias_correction', action="store_true", help='Providing Cufflinks with a multifasta file via this option instructs it to run our new bias detection and correction algorithm which can significantly improve accuracy of transcript abundance estimates.') - parser.add_option( '', '--dbkey', dest='dbkey', help='The build of the reference dataset' ) - parser.add_option( '', '--index_dir', dest='index_dir', help='GALAXY_DATA_INDEX_DIR' ) + parser.add_option( '', '--index', dest='index', help='The path of the reference genome' ) parser.add_option( '', '--ref_file', dest='ref_file', help='The reference dataset from the history' ) # Outputs. @@ -123,21 +108,16 @@ def __main__(): # If doing bias correction, set/link to sequence file. if options.do_bias_correction: - if options.ref_file != 'None': + if options.ref_file: # Sequence data from history. # Create symbolic link to ref_file so that index will be created in working directory. seq_path = "ref.fa" os.symlink( options.ref_file, seq_path ) else: # Sequence data from loc file. - cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' ) - if not os.path.exists( cached_seqs_pointer_file ): - stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file ) - # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa, - # and the equCab2.fa file will contain fasta sequences. - seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file ) - if seq_path == '': - stop_err( 'No sequence data found for dbkey %s, so bias correction cannot be used.' % options.dbkey ) + if not os.path.exists( options.index ): + stop_err( 'Reference genome %s not present, request it by reporting this error.' % options.index ) + seq_path = options.index # Build command. diff --git a/tools/ngs_rna/cuffdiff_wrapper.xml b/tools/ngs_rna/cuffdiff_wrapper.xml index 98241b06898..c9ba78f7404 100644 --- a/tools/ngs_rna/cuffdiff_wrapper.xml +++ b/tools/ngs_rna/cuffdiff_wrapper.xml @@ -1,4 +1,4 @@ - + find significant changes in transcript expression, splicing, and promoter use @@ -42,14 +42,12 @@ ## Bias correction? #if $bias_correction.do_bias_correction == "Yes": - -b + -b #if $bias_correction.seq_source.index_source == "history": --ref_file=$bias_correction.seq_source.ref_file #else: - --ref_file="None" + --index=${bias_correction.seq_source.index.fields.path} #end if - --dbkey=${gtf_input.metadata.dbkey} - --index_dir=${GALAXY_DATA_INDEX_DIR} #end if ## Inputs. @@ -131,7 +129,14 @@ - + + + + + + + + diff --git a/tools/ngs_rna/cufflinks_wrapper.py b/tools/ngs_rna/cufflinks_wrapper.py index 2387bce4bf3..ac91c3604c4 100644 --- a/tools/ngs_rna/cufflinks_wrapper.py +++ b/tools/ngs_rna/cufflinks_wrapper.py @@ -10,20 +10,6 @@ def stop_err( msg ): sys.stderr.write( "%s\n" % msg ) sys.exit() -# Copied from sam_to_bam.py: -def check_seq_file( dbkey, cached_seqs_pointer_file ): - seq_path = '' - for line in open( cached_seqs_pointer_file ): - line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#' ) and line.startswith( 'index' ): - fields = line.split( '\t' ) - if len( fields ) < 3: - continue - if fields[1] == dbkey: - seq_path = fields[2].strip() - break - return seq_path - def __main__(): #Parse Command Line parser = optparse.OptionParser() @@ -52,8 +38,7 @@ def __main__(): # Bias correction options. parser.add_option( '-b', dest='do_bias_correction', action="store_true", help='Providing Cufflinks with a multifasta file via this option instructs it to run our new bias detection and correction algorithm which can significantly improve accuracy of transcript abundance estimates.') - parser.add_option( '', '--dbkey', dest='dbkey', help='The build of the reference dataset' ) - parser.add_option( '', '--index_dir', dest='index_dir', help='GALAXY_DATA_INDEX_DIR' ) + parser.add_option( '', '--index', dest='index', help='The path of the reference genome' ) parser.add_option( '', '--ref_file', dest='ref_file', help='The reference dataset from the history' ) # Global model. @@ -82,21 +67,16 @@ def __main__(): # If doing bias correction, set/link to sequence file. if options.do_bias_correction: - if options.ref_file != 'None': + if options.ref_file: # Sequence data from history. # Create symbolic link to ref_file so that index will be created in working directory. seq_path = "ref.fa" os.symlink( options.ref_file, seq_path ) else: # Sequence data from loc file. - cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' ) - if not os.path.exists( cached_seqs_pointer_file ): - stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file ) - # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa, - # and the equCab2.fa file will contain fasta sequences. - seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file ) - if seq_path == '': - stop_err( 'No sequence data found for dbkey %s, so bias correction cannot be used.' % options.dbkey ) + if not os.path.exists( options.index ): + stop_err( 'Reference genome %s not present, request it by reporting this error.' % options.index ) + seq_path = options.index # Build command. diff --git a/tools/ngs_rna/cufflinks_wrapper.xml b/tools/ngs_rna/cufflinks_wrapper.xml index 29826a33108..cc6a12532af 100644 --- a/tools/ngs_rna/cufflinks_wrapper.xml +++ b/tools/ngs_rna/cufflinks_wrapper.xml @@ -1,4 +1,4 @@ - + transcript assembly and FPKM (RPKM) estimates for RNA-Seq data @@ -28,14 +28,12 @@ ## Bias correction? #if $bias_correction.do_bias_correction == "Yes": - -b + -b #if $bias_correction.seq_source.index_source == "history": --ref_file=$bias_correction.seq_source.ref_file #else: - --ref_file="None" + --index=${bias_correction.seq_source.index.fields.path} #end if - --dbkey=${input.metadata.dbkey} - --index_dir=${GALAXY_DATA_INDEX_DIR} #end if ## Multi-read correct? @@ -66,15 +64,15 @@ - - + + - + - + @@ -82,7 +80,14 @@ - + + + + + + + + diff --git a/tools/ngs_rna/cuffmerge_wrapper.py b/tools/ngs_rna/cuffmerge_wrapper.py index 6d7fc39d08d..ae2c6cef849 100644 --- a/tools/ngs_rna/cuffmerge_wrapper.py +++ b/tools/ngs_rna/cuffmerge_wrapper.py @@ -8,20 +8,6 @@ def stop_err( msg ): sys.stderr.write( '%s\n' % msg ) sys.exit() -# Copied from sam_to_bam.py: -def check_seq_file( dbkey, cached_seqs_pointer_file ): - seq_path = '' - for line in open( cached_seqs_pointer_file ): - line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#' ) and line.startswith( 'index' ): - fields = line.split( '\t' ) - if len( fields ) < 3: - continue - if fields[1] == dbkey: - seq_path = fields[2].strip() - break - return seq_path - def __main__(): #Parse Command Line parser = optparse.OptionParser() @@ -31,8 +17,7 @@ def __main__(): # Wrapper / Galaxy options. - parser.add_option( '', '--dbkey', dest='dbkey', help='The build of the reference dataset' ) - parser.add_option( '', '--index_dir', dest='index_dir', help='GALAXY_DATA_INDEX_DIR' ) + parser.add_option( '', '--index', dest='index', help='The path of the reference genome' ) parser.add_option( '', '--ref_file', dest='ref_file', help='The reference dataset from the history' ) # Outputs. @@ -61,21 +46,16 @@ def __main__(): # Set/link to sequence file. if options.use_seq_data: - if options.ref_file != 'None': + if options.ref_file: # Sequence data from history. # Create symbolic link to ref_file so that index will be created in working directory. seq_path = "ref.fa" os.symlink( options.ref_file, seq_path ) else: # Sequence data from loc file. - cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' ) - if not os.path.exists( cached_seqs_pointer_file ): - stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file ) - # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa, - # and the equCab2.fa file will contain fasta sequences. - seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file ) - if seq_path == '': - stop_err( 'No sequence data found for dbkey %s, so sequence data cannot be used.' % options.dbkey ) + if not os.path.exists( options.index ): + stop_err( 'Reference genome %s not present, request it by reporting this error.' % options.index ) + seq_path = options.index # Build command. diff --git a/tools/ngs_rna/cuffmerge_wrapper.xml b/tools/ngs_rna/cuffmerge_wrapper.xml index 09a687b468e..0209de52308 100644 --- a/tools/ngs_rna/cuffmerge_wrapper.xml +++ b/tools/ngs_rna/cuffmerge_wrapper.xml @@ -1,4 +1,4 @@ - + merge together several Cufflinks assemblies @@ -16,14 +16,12 @@ ## Use sequence data? #if $seq_data.use_seq_data == "Yes": - -s + -s #if $seq_data.seq_source.index_source == "history": --ref_file=$seq_data.seq_source.ref_file #else: - --ref_file="None" + --index=${seq_data.seq_source.index.fields.path} #end if - --dbkey=${first_input.metadata.dbkey} - --index_dir=${GALAXY_DATA_INDEX_DIR} #end if ## Outputs. @@ -64,7 +62,14 @@ - + + + + + + + + From 70996cfb3dab377bbba4de457376078b4939dc3c Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 18 Apr 2013 19:25:55 +0200 Subject: [PATCH 009/318] Add new loc file for SAMtools indexes to support genome variants --- tool-data/sam_fa_new_indices.loc.sample | 30 +++++++++++++++++++++++++ tool_data_table_conf.xml.sample | 20 +++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tool-data/sam_fa_new_indices.loc.sample diff --git a/tool-data/sam_fa_new_indices.loc.sample b/tool-data/sam_fa_new_indices.loc.sample new file mode 100644 index 00000000000..ddff8d4c46a --- /dev/null +++ b/tool-data/sam_fa_new_indices.loc.sample @@ -0,0 +1,30 @@ +#This is a sample file distributed with Galaxy that enables tools +#to use a directory of Samtools indexed sequences data files. You will need +#to create these data files and then create a sam_fa_new_indices.loc file +#similar to this one (store it in this directory) that points to +#the directories in which those files are stored. The sam_fa_new_indices.loc +#file has this format (white space characters are TAB characters): +# +# +# +#So, for example, if you had hg19 Canonical indexed stored in +# +# /depot/data2/galaxy/hg19/sam/, +# +#then the sam_fa_new_indices.loc entry would look like this: +# +#hg19canon hg19 Human (Homo sapiens): hg19 Canonical /depot/data2/galaxy/hg19/sam/hg19canon.fa +# +#and your /depot/data2/galaxy/hg19/sam/ directory +#would contain hg19canon.fa and hg19canon.fa.fai files. +# +#Your sam_fa_new_indices.loc file should include an entry per line for +#each index set you have stored. The file in the path does actually +#exist, but it should never be directly used. Instead, the name serves +#as a prefix for the index file. For example: +# +#hg18canon hg18 Human (Homo sapiens): hg18 Canonical /depot/data2/galaxy/hg18/sam/hg18canon.fa +#hg18full hg18 Human (Homo sapiens): hg18 Full /depot/data2/galaxy/hg18/sam/hg18full.fa +#hg19canon hg19 Human (Homo sapiens): hg19 Canonical /depot/data2/galaxy/hg19/sam/hg19canon.fa +#hg19full hg19 Human (Homo sapiens): hg19 Full /depot/data2/galaxy/hg19/sam/hg19full.fa + diff --git a/tool_data_table_conf.xml.sample b/tool_data_table_conf.xml.sample index 5ea328f1bc6..b120b685e57 100644 --- a/tool_data_table_conf.xml.sample +++ b/tool_data_table_conf.xml.sample @@ -55,10 +55,26 @@ value, name, path - - + +
+ value, dbkey, name, path +
From 2bdc8574d181806086cb6fdb97b8ad6c167d5d70 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 20 Apr 2013 01:33:57 -0500 Subject: [PATCH 010/318] Implement dynamic toolbox filters. Overview: Right now these filters are implemented via python code, but a higher level configuration mechanism could be implemented on top of this one easily. Added the well documented file lib/galaxy/tools/filters/examples.py.sample that outlines the implementations of some possible filters and how to configure them in universe_wsgi.ini. Implementation: Refactored Toolbox.to_dict and remove Tool._is_hidden_for_user, moving and unifying all filter building functionality into tools/filters/__init.py. Implemented new dynamic filters for tools, labels, and sections. Multiple such filters can be specified by giving module (relative to galaxy.tools.filters) and function names for each as comma seperated list to the respective options tool_filters, tool_label_filters, and tool_section_filters in universe_wsgi.ini. Explicit example lines to add to universe_wsgi.ini can be found in examples.py.sample (described above). WARNING: This mechanism only controls what tools are visible to the user. I imagine a savy user could call any tool by building custom requests or utilizing the API, to truely block users from accessing certain tools I recommend using the dynamic job runners. --- lib/galaxy/config.py | 3 + lib/galaxy/tools/__init__.py | 131 ++++++++++---------- lib/galaxy/tools/filters/__init__.py | 67 ++++++++++ lib/galaxy/tools/filters/examples.py.sample | 87 +++++++++++++ 4 files changed, 220 insertions(+), 68 deletions(-) create mode 100644 lib/galaxy/tools/filters/__init__.py create mode 100644 lib/galaxy/tools/filters/examples.py.sample diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 6e4cc731b9b..8393d7cfe8e 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -64,6 +64,9 @@ class Configuration( object ): tcf = kwargs[ 'tool_config_files' ] else: tcf = 'tool_conf.xml' + self.tool_filters = listify( kwargs.get( "tool_filters", [] ) ) + self.tool_label_filters = listify( kwargs.get( "tool_label_filters", [] ) ) + self.tool_section_filters = listify( kwargs.get( "tool_section_filters", [] ) ) self.tool_configs = [ resolve_path( p, self.root ) for p in listify( tcf ) ] self.tool_data_table_config_path = resolve_path( kwargs.get( 'tool_data_table_config_path', 'tool_data_table_conf.xml' ), self.root ) self.shed_tool_data_table_config = resolve_path( kwargs.get( 'shed_tool_data_table_config', 'shed_tool_data_table_conf.xml' ), self.root ) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 66b900d413a..f8fbe3d81b5 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -47,6 +47,7 @@ from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repea from galaxy.tools.parameters.input_translation import ToolInputTranslator from galaxy.tools.parameters.output import ToolOutputActionGroup from galaxy.tools.parameters.validation import LateValidationError +from galaxy.tools.filters import FilterFactory from galaxy.tools.test import ToolTestBuilder from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object, xml_text, xml_to_string from galaxy.util.bunch import Bunch @@ -119,6 +120,7 @@ class ToolBox( object ): # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the tag. self.tool_root_dir = tool_root_dir self.app = app + self.filter_factory = FilterFactory( self ) self.init_dependency_manager() config_filenames = listify( config_filenames ) for config_filename in config_filenames: @@ -679,76 +681,19 @@ class ToolBox( object ): """ return self.app.model.context - def to_dict( self, trans, in_panel=True, trackster=False ): - - def filter_for_panel( item, filters ): - """ - Filters tool panel elements so that only those that are compatible - with provided filters are kept. - """ - def _apply_filter( filter_item, filter_list ): - for filter_method in filter_list: - if not filter_method( filter_item ): - return False - return True - if isinstance( item, Tool ): - if _apply_filter( item, filters[ 'tool' ] ): - return item - elif isinstance( item, ToolSectionLabel ): - if _apply_filter( item, filters[ 'label' ] ): - return item - elif isinstance( item, ToolSection ): - # Filter section item-by-item. Only show a label if there are - # non-filtered tools below it. - - if _apply_filter( item, filters[ 'section' ] ): - cur_label_key = None - tools_under_label = False - filtered_elems = item.elems.copy() - for key, section_item in item.elems.items(): - if isinstance( section_item, Tool ): - # Filter tool. - if _apply_filter( section_item, filters[ 'tool' ] ): - tools_under_label = True - else: - del filtered_elems[ key ] - elif isinstance( section_item, ToolSectionLabel ): - # If there is a label and it does not have tools, - # remove it. - if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ): - del filtered_elems[ cur_label_key ] - - # Reset attributes for new label. - cur_label_key = key - tools_under_label = False - - - # Handle last label. - if cur_label_key and not tools_under_label: - del filtered_elems[ cur_label_key ] - - # Only return section if there are elements. - if len( filtered_elems ) != 0: - copy = item.copy() - copy.elems = filtered_elems - return copy - - return None - + def to_dict( self, trans, in_panel=True, **kwds ): # # Dictify toolbox. # - + context = Bunch( toolbox=self, trans=trans, **kwds ) if in_panel: panel_elts = [ val for val in self.tool_panel.itervalues() ] - # Filter if necessary. - filters = dict( tool=[ lambda x: not x._is_hidden_for_user( trans.user ) ], section=[], label=[] ) #hidden tools filter - if trackster: - filters[ 'tool' ].append( lambda x: x.trackster_conf ) # If tool has a trackster config, it can be used in Trackster. + filters = self.filter_factory.build_filters( trans, **kwds ) + filtered_panel_elts = [] for index, elt in enumerate( panel_elts ): - elt = filter_for_panel( elt, filters ) + elt = _filter_for_panel( elt, filters, context ) if elt: filtered_panel_elts.append( elt ) panel_elts = filtered_panel_elts @@ -759,13 +704,68 @@ class ToolBox( object ): rval.append( elt.to_dict( trans, for_link=True ) ) else: tools = [] - for id, tool in self.app.toolbox.tools_by_id.items(): + for id, tool in self.toolbox.tools_by_id.items(): tools.append( tool.to_dict( trans ) ) rval = tools return rval +def _filter_for_panel( item, filters, context ): + """ + Filters tool panel elements so that only those that are compatible + with provided filters are kept. + """ + def _apply_filter( filter_item, filter_list ): + for filter_method in filter_list: + if not filter_method( context, filter_item ): + return False + return True + if isinstance( item, Tool ): + if _apply_filter( item, filters[ 'tool' ] ): + return item + elif isinstance( item, ToolSectionLabel ): + if _apply_filter( item, filters[ 'label' ] ): + return item + elif isinstance( item, ToolSection ): + # Filter section item-by-item. Only show a label if there are + # non-filtered tools below it. + + if _apply_filter( item, filters[ 'section' ] ): + cur_label_key = None + tools_under_label = False + filtered_elems = item.elems.copy() + for key, section_item in item.elems.items(): + if isinstance( section_item, Tool ): + # Filter tool. + if _apply_filter( section_item, filters[ 'tool' ] ): + tools_under_label = True + else: + del filtered_elems[ key ] + elif isinstance( section_item, ToolSectionLabel ): + # If there is a label and it does not have tools, + # remove it. + if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ): + del filtered_elems[ cur_label_key ] + + # Reset attributes for new label. + cur_label_key = key + tools_under_label = False + + # Handle last label. + if cur_label_key and not tools_under_label: + del filtered_elems[ cur_label_key ] + + # Only return section if there are elements. + if len( filtered_elems ) != 0: + copy = item.copy() + copy.elems = filtered_elems + return copy + + return None + + + class ToolSection( object ): """ A group of tools with similar type/purpose that will be displayed as a @@ -2897,11 +2897,6 @@ class Tool( object ): self.sa_session.flush() return primary_datasets - def _is_hidden_for_user( self, user ): - if self.hidden or ( not user and self.require_login ): - return True - return False - def to_dict( self, trans, for_link=False, for_display=False ): """ Returns dict of tool. """ diff --git a/lib/galaxy/tools/filters/__init__.py b/lib/galaxy/tools/filters/__init__.py new file mode 100644 index 00000000000..db101e0f3b5 --- /dev/null +++ b/lib/galaxy/tools/filters/__init__.py @@ -0,0 +1,67 @@ + + +class FilterFactory( object ): + """ + An instance of this class is responsible for filtering the list + of tools presented to a given user in a given context. + """ + + def __init__( self, toolbox ): + self.toolbox = toolbox + + # Prepopulate dict containing filters that are always checked, + # other filters that get checked depending on context (e.g. coming from + # trackster or no user found are added in build filters). + self.default_filters = dict( tool=[ _not_hidden ], section=[], label=[] ) + # Add dynamic filters to these default filters. + config = toolbox.app.config + self.__init_filters( "tool", config.tool_filters ) + self.__init_filters( "section", config.tool_section_filters ) + self.__init_filters( "label", config.tool_label_filters ) + + def build_filters( self, trans, **kwds ): + """ + Build list of filters to check tools against given current context. + """ + filters = self.default_filters.copy() + + if not trans.user: + filters[ "tool" ].append( _requires_login ) + + if kwds.get( "trackster", False ): + filters[ "tool" ].append( _has_trackster_conf ) + + return filters + + def __init_filters( self, key, filters ): + for filter in filters: + filter_function = self.__build_filter_function( filter ) + self.default_filters[ key ].append( filter_function ) + + def __build_filter_function( self, filter_name ): + """Obtain python function (importing a submodule if needed) + corresponding to filter_name. + """ + if ":" in filter_name: + # Should be a submodule of filters (e.g. examples:restrict_development_tools) + (module_name, function_name) = filter_name.rsplit(":", 1) + module = __import__( module_name, globals() ) + function = getattr( module, function_name ) + else: + # No module found, just load a function from this file or + # one that has be explicitly imported. + function = getattr( globals(), filter_name ) + return function + + +## Stock Filter Functions +def _not_hidden( context, tool ): + return not tool.hidden + + +def _requires_login( context, tool ): + return not tool.require_login + + +def _has_trackster_conf( context, tool ): + return tool.trackster_conf diff --git a/lib/galaxy/tools/filters/examples.py.sample b/lib/galaxy/tools/filters/examples.py.sample new file mode 100644 index 00000000000..0fa0240ae47 --- /dev/null +++ b/lib/galaxy/tools/filters/examples.py.sample @@ -0,0 +1,87 @@ +import logging +log = logging.getLogger( __name__ ) + + +def restrict_upload_to_admins( context, tool ): + """ + This tool filter will hide the upload tool from all users except admin + users. This can be enabled by renaming this file to examples.py and adding + the following to the ``app:main`` section of ``universe_wsgi.ini``: + + tool_filters = examples:restrict_upload_to_admins + """ + if tool.name == "Upload File": + return context.trans.user_is_admin() + return True + + +def disable_gatk( context, tool ): + """ + This tool filter will disable all gatk tools when enabled. This can be + enabled by renaming this file to examples.py and adding the following to the + ``app:main`` section of ``universe_wsgi.ini``: + + tool_filters = examples:disable_gatk + """ + return not any( [ requirement.name == "gatk" for requirement in tool.requirements ] ) + + +def explicit_user_mapping( context, section ): + """ + This tool section filter uses an explicit mapping to describe what users can view + which tool sections. Anonymous users will only be able to view the "Get Data" + tool section (with id getext). This can be enabled by renaming this file to + examples.py and adding the following to the ``app:main`` section of + ``universe_wsgi.ini``: + + tool_section_filters = examples:explicit_user_mapping + """ + users_sections = { + None: [ "getext" ], + "bob@example.com": [ "getext", "textutil", "filter" ], + "mary@example.com": [ "getext", "textutil", "filter", "ngs" ], + } + user = context.trans.user + email = user and user.email + valid_sections = users_sections.get( email, [] ) + return section.id in valid_sections + + +DEVELOPERS = [ "mary@example.com" ] + + +def restrict_development_tools( context, tool ): + """ + This tool filter will disable all tools with the string alpha appearing in + the version for all users except those explicitly appearing in the DEVELOPERS list + defined above. This can be enabled by renaming this file to examples.py and + adding the following to the ``app:main`` section of ``universe_wsgi.ini``: + + tool_filters = examples:restrict_development_tools + """ + version = tool.version + user = context.trans.user + email = user and user.email + return "alpha" not in version or email in DEVELOPERS + + +def per_host_tool_sections( context, section ): + """ + This tool section filter results in different sections being display based on + the URL the user is making the request to. This could allow a single Galaxy instance + to seem like several different instances hosting different tools based on the URL used + to access the Galxy. This can be enabled by renaming this file to examples.py and adding + the following to the ``app:main`` section of ``universe_wsgi.ini``: + + tool_section_filters = examples:per_host_tool_sections + """ + host = context.trans.request.host + # Core tools used by all virtual hosts. + valid_sections = [ "getext", "textutil", "filter" ] + if "ngs.galaxy.example.com" in host: + valid_sections += [ "ngs" ] + elif "microarray.galaxy.example.com" in host: + valid_sections += [ "microarray" ] + elif "proteomics.galaxy.example.com" in host: + valid_sections += [ "proteomics" ] + return section.id in valid_sections From ada6eacee3a1819e6527571c169994a981a5aa66 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 20 Apr 2013 11:10:23 -0500 Subject: [PATCH 011/318] Dynamic Toolbox Filtering: Allow whitespace around commas between filters in configuration file. --- lib/galaxy/tools/filters/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/filters/__init__.py b/lib/galaxy/tools/filters/__init__.py index db101e0f3b5..a99a94d7454 100644 --- a/lib/galaxy/tools/filters/__init__.py +++ b/lib/galaxy/tools/filters/__init__.py @@ -45,12 +45,12 @@ class FilterFactory( object ): if ":" in filter_name: # Should be a submodule of filters (e.g. examples:restrict_development_tools) (module_name, function_name) = filter_name.rsplit(":", 1) - module = __import__( module_name, globals() ) - function = getattr( module, function_name ) + module = __import__( module_name.strip(), globals() ) + function = getattr( module, function_name.strip() ) else: # No module found, just load a function from this file or # one that has be explicitly imported. - function = getattr( globals(), filter_name ) + function = getattr( globals(), filter_name.strip() ) return function From 88b06c8485b89ca02c77cb1761b0f6a5ba333881 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Mon, 20 May 2013 15:51:10 -0400 Subject: [PATCH 012/318] Change the tool shed grid ToolsFunctionallyCorrectColumn column type. --- lib/tool_shed/grids/repository_grids.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tool_shed/grids/repository_grids.py b/lib/tool_shed/grids/repository_grids.py index a8c0ed31114..a2831e1b3b3 100644 --- a/lib/tool_shed/grids/repository_grids.py +++ b/lib/tool_shed/grids/repository_grids.py @@ -121,7 +121,7 @@ class RepositoryGrid( grids.Grid ): return escape_html( repository.revision( trans.app ) ) - class ToolsFunctionallyCorrectColumn( grids.BooleanColumn ): + class ToolsFunctionallyCorrectColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): # This column will display the value associated with the currently displayed metadata revision. @@ -1130,7 +1130,7 @@ class RepositoryMetadataGrid( grids.Grid ): return '' - class ToolsFunctionallyCorrectColumn( grids.BooleanColumn ): + class ToolsFunctionallyCorrectColumn( grids.TextColumn ): def get_value( self, trans, grid, repository ): # This column will display the value associated with the currently displayed metadata revision. From eacaeb97989b1746ae6aed1db10aa272e041322a Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 21 May 2013 10:39:47 -0400 Subject: [PATCH 013/318] Fix for uploading toolshed repository via hg. --- lib/galaxy/webapps/tool_shed/controllers/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/tool_shed/controllers/upload.py b/lib/galaxy/webapps/tool_shed/controllers/upload.py index 55ed8838f1a..c273f2fcd3b 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/upload.py +++ b/lib/galaxy/webapps/tool_shed/controllers/upload.py @@ -74,7 +74,7 @@ class UploadController( BaseUIController ): fd, uploaded_file_name = tempfile.mkstemp() uploaded_file = open( uploaded_file_name, 'wb' ) while 1: - chunk = stream.read( CHUNK_SIZE ) + chunk = stream.read( util.CHUNK_SIZE ) if not chunk: break uploaded_file.write( chunk ) From 219f774db677511ef38baac656aebb5d77f808b6 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 21 May 2013 12:15:17 -0400 Subject: [PATCH 014/318] Add an option to install and test every installable changeset revision of every repository returned by the tool shed API call. Default behavior changed to only install and test the most recent installable changeset revision of each repository. --- .../functional_tests.py | 115 ++++++++++-------- 1 file changed, 65 insertions(+), 50 deletions(-) diff --git a/test/install_and_test_tool_shed_repositories/functional_tests.py b/test/install_and_test_tool_shed_repositories/functional_tests.py index 913a7c7b0cc..a79fbe21f27 100644 --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -225,13 +225,25 @@ def get_api_url( base, parts=[], params=None, key=None ): url += '&%s' % params return url +def get_latest_downloadable_changeset_revision( url, name, owner ): + api_url_parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] + params = urllib.urlencode( dict( name=name, owner=owner ) ) + api_url = get_api_url( url, api_url_parts, params ) + changeset_revisions = json_from_url( api_url ) + if changeset_revisions: + return changeset_revisions[ -1 ] + else: + return '000000000000' + def get_repository_info_from_api( url, repository_info_dict ): parts = [ 'api', 'repositories', repository_info_dict[ 'repository_id' ] ] api_url = get_api_url( base=url, parts=parts ) extended_dict = json_from_url( api_url ) + latest_changeset_revision = get_latest_downloadable_changeset_revision( url, extended_dict[ 'name' ], extended_dict[ 'owner' ] ) + extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) return extended_dict -def get_repositories_to_install( location, source='file', format='json' ): +def get_repositories_to_install( tool_shed_url, latest_revision_only=True ): ''' Get a list of repository info dicts to install. This method expects a json list of dicts with the following structure: [ @@ -246,24 +258,49 @@ def get_repositories_to_install( location, source='file', format='json' ): ] NOTE: If the tool shed URL specified in any dict is not present in the tool_sheds_conf.xml, the installation will fail. ''' - if source == 'file': - listing = file( location, 'r' ).read() - elif source == 'url': - assert tool_shed_api_key is not None, 'Cannot proceed without tool shed API key.' - params = urllib.urlencode( dict( do_not_test='false', - downloadable='true', - malicious='false', - includes_tools='true', - skip_tool_test='false' ) ) - api_url = get_api_url( base=location, parts=[ 'repository_revisions' ], params=params ) - if format == 'json': - return json_from_url( api_url ) + assert tool_shed_api_key is not None, 'Cannot proceed without tool shed API key.' + params = urllib.urlencode( dict( do_not_test='false', + downloadable='true', + malicious='false', + includes_tools='true', + skip_tool_test='false' ) ) + api_url = get_api_url( base=tool_shed_url, parts=[ 'repository_revisions' ], params=params ) + base_repository_list = json_from_url( api_url ) + known_repository_ids = {} + detailed_repository_list = [] + for repository_to_install_dict in base_repository_list: + # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the + # module that will generate the install methods. + repository_info_dict = get_repository_info_from_api( galaxy_tool_shed_url, repository_to_install_dict ) + if repository_info_dict[ 'latest_revision' ] == '000000000000': + continue + owner = repository_info_dict[ 'owner' ] + name = repository_info_dict[ 'name' ] + changeset_revision = repository_to_install_dict[ 'changeset_revision' ] + repository_id = repository_to_install_dict[ 'repository_id' ] + # We are testing deprecated repositories, because it is possible that a deprecated repository contains valid + # and functionally correct tools that someone has previously installed. Deleted repositories have never been installed, + # and therefore do not need to be checked. If they are undeleted, this script will then test them the next time it runs. + if repository_info_dict[ 'deleted' ]: + log.info( "Skipping revision %s of repository id %s (%s/%s) since the repository is deleted...", + changeset_revision, + repository_id, + name, + owner ) + continue + # Now merge the dict returned from /api/repository_revisions with the detailed dict we just retrieved. + if latest_revision_only: + if changeset_revision == repository_info_dict[ 'latest_revision' ]: + detailed_repository_list.append( dict( repository_info_dict.items() + repository_to_install_dict.items() ) ) + else: + detailed_repository_list.append( dict( repository_info_dict.items() + repository_to_install_dict.items() ) ) + repositories_tested = len( detailed_repository_list ) + if latest_revision_only: + skipped_previous = ' and metadata revisions that are not the most recent' else: - raise AssertionError( 'Do not know how to handle source type %s.' % source ) - if format == 'json': - return from_json_string( listing ) - else: - raise AssertonError( 'Unknown format %s.' % format ) + skipped_previous = '' + log.info( 'After removing deleted repositories%s from the list, %d remain to be tested.', skipped_previous, repositories_tested ) + return detailed_repository_list def get_tool_info_from_test_id( test_id ): ''' @@ -286,13 +323,7 @@ def get_tool_test_results_from_api( tool_shed_url, metadata_revision_id ): return tool_test_results def is_latest_downloadable_revision( url, repository_info_dict ): - api_url_parts = [ 'api', 'repositories', 'get_ordered_installable_revisions' ] - params = urllib.urlencode( dict( name=repository_info_dict[ 'name' ], owner=repository_info_dict[ 'owner' ] ) ) - api_url = get_api_url( url, api_url_parts, params ) - changeset_revisions = json_from_url( api_url ) - # The get_ordered_installable_revisions returns a list of changeset hashes, with the last hash in the list - # being the most recent installable revision. - latest_revision = changeset_revisions.pop() + latest_revision = get_latest_downloadable_changeset_revision( url, name=repository_info_dict[ 'name' ], owner=repository_info_dict[ 'owner' ] ) return str( repository_info_dict[ 'changeset_revision' ] ) == str( latest_revision ) def json_from_url( url ): @@ -532,41 +563,25 @@ def main(): shed_tool_data_table_config=None, persist=False ) # Initialize some variables for the summary that will be printed to stdout. - repositories_tested = 0 repositories_passed = [] repositories_failed = [] repositories_failed_install = [] try: - detailed_repository_list = [] # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. log.info( "Retrieving repositories to install from the URL:\n%s\n", str( galaxy_tool_shed_url ) ) - repositories_to_install = get_repositories_to_install( galaxy_tool_shed_url, source='url' ) + if '-check_all_revisions' not in sys.argv: + repositories_to_install = get_repositories_to_install( galaxy_tool_shed_url, latest_revision_only=True ) + else: + repositories_to_install = get_repositories_to_install( galaxy_tool_shed_url, latest_revision_only=False ) log.info( "Retrieved %d repositories from the API.", len( repositories_to_install ) ) - for repository_to_install_dict in repositories_to_install: - # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the - # module that will generate the install methods. - repository_info_dict = get_repository_info_from_api( galaxy_tool_shed_url, repository_to_install_dict ) - # We are testing deprecated repositories, because it is possible that a deprecated repository contains valid - # and functionally correct tools that someone has previously installed. Deleted repositories have never been installed, - # and therefore do not need to be checked. If they are undeleted, this script will then test them the next time it runs. - if repository_info_dict[ 'deleted' ]: - log.info( "Skipping revision %s of repository id %s (%s/%s) since the repository is deleted...", - repository_to_install_dict[ 'changeset_revision' ], - repository_to_install_dict[ 'repository_id' ], - repository_info_dict[ 'owner' ], - repository_info_dict[ 'name' ] ) - continue - # Now merge the dict returned from /api/repository_revisions with the detailed dict we just retrieved. - detailed_repository_list.append( dict( repository_info_dict.items() + repository_to_install_dict.items() ) ) - repositories_tested = len( detailed_repository_list ) - log.info( 'After removing deleted repositories from the list, %d remain to be tested.', repositories_tested ) if '-list_repositories' in sys.argv: log.info( "The API returned the following repositories, not counting deleted:" ) - for repository_info_dict in detailed_repository_list: + for repository_info_dict in repositories_to_install: log.info( "%s owned by %s changeset revision %s", repository_info_dict.get( 'name', None ), repository_info_dict.get( 'owner', None ), repository_info_dict.get( 'changeset_revision', None ) ) + repositories_tested = len( repositories_to_install ) # This loop will iterate through the list of repositories generated by the above code, having already filtered out any # that were marked as deleted. For each repository, it will generate a test method that will use Twill to install that # repository into the embedded Galaxy application that was started up, selecting to install repository and tool @@ -575,7 +590,7 @@ def main(): # it will record the result of the tests, and if any failed, the traceback and captured output of the tool that was run. # After all tests have completed, the repository is uninstalled, so that the previous test cases don't interfere with # the next repository's functional tests. - for repository_info_dict in detailed_repository_list: + for repository_info_dict in repositories_to_install: """ Each repository_info_dict looks something like: { @@ -600,8 +615,8 @@ def main(): """ repository_status = dict() params = dict() - repository_id = repository_info_dict.get( 'repository_id', None ) - changeset_revision = repository_info_dict.get( 'changeset_revision', None ) + repository_id = str( repository_info_dict.get( 'repository_id', None ) ) + changeset_revision = str( repository_info_dict.get( 'changeset_revision', None ) ) metadata_revision_id = repository_info_dict.get( 'id', None ) # Add the URL for the tool shed we're installing from, so the automated installation methods go to the right place. repository_info_dict[ 'tool_shed_url' ] = galaxy_tool_shed_url From 290659d16de88e126c4cc9dfd1dad60d817249ba Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 21 May 2013 16:02:21 -0400 Subject: [PATCH 015/318] Add the ability to define a list of repositories or changeset revisions of repositories that should not be tested by the automated testing framework. --- .../functional_tests.py | 127 +++++++++++++++--- 1 file changed, 106 insertions(+), 21 deletions(-) diff --git a/test/install_and_test_tool_shed_repositories/functional_tests.py b/test/install_and_test_tool_shed_repositories/functional_tests.py index a79fbe21f27..7ca68b0aa8c 100644 --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -129,6 +129,7 @@ tool_data_table_conf_xml_template = ''' galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', None ) tool_shed_api_key = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY', None ) +exclude_list_file = os.environ.get( 'GALAXY_INSTALL_TEST_EXCLUDE_REPOSITORIES', 'install_test_exclude.xml' ) if tool_shed_api_key is None: print "This script requires the GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY environment variable to be set and non-empty." @@ -243,6 +244,13 @@ def get_repository_info_from_api( url, repository_info_dict ): extended_dict[ 'latest_revision' ] = str( latest_changeset_revision ) return extended_dict +def get_repository_tuple_from_elem( elem ): + attributes = elem.attrib + name = attributes.get( 'name', None ) + owner = attributes.get( 'owner', None ) + changeset_revision = attributes.get( 'changeset_revision', None ) + return ( name, owner, changeset_revision ) + def get_repositories_to_install( tool_shed_url, latest_revision_only=True ): ''' Get a list of repository info dicts to install. This method expects a json list of dicts with the following structure: @@ -266,6 +274,7 @@ def get_repositories_to_install( tool_shed_url, latest_revision_only=True ): skip_tool_test='false' ) ) api_url = get_api_url( base=tool_shed_url, parts=[ 'repository_revisions' ], params=params ) base_repository_list = json_from_url( api_url ) + log.info( 'The api returned %d metadata revisions.', len( base_repository_list ) ) known_repository_ids = {} detailed_repository_list = [] for repository_to_install_dict in base_repository_list: @@ -336,6 +345,49 @@ def json_from_url( url ): raise return parsed_json +def parse_exclude_list( xml_filename ): + ''' + This method should return a list with the following structure: + [ + { + 'reason': The default reason or the reason specified in this section, + 'repositories': + [ + ( name, owner, changeset revision if changeset revision else None ), + ( name, owner, changeset revision if changeset revision else None ) + ] + }, + { + 'reason': The default reason or the reason specified in this section, + 'repositories': + [ + ( name, owner, changeset revision if changeset revision else None ), + ( name, owner, changeset revision if changeset revision else None ) + ] + }, + ] + ''' + exclude_list = [] + xml_tree = parse_xml( xml_filename ) + tool_sheds = xml_tree.findall( 'repositories' ) + xml_element = None + for tool_shed in tool_sheds: + if galaxy_tool_shed_url != tool_shed.attrib[ 'tool_shed' ]: + continue + else: + xml_element = tool_shed + for reason_section in xml_element: + print reason_section + reason_text = reason_section.find( 'text' ).text + repositories = reason_section.findall( 'repository' ) + exclude_dict = dict( reason=reason_text, repositories=[] ) + for repository in repositories: + repository_tuple = get_repository_tuple_from_elem( repository ) + if repository_tuple not in exclude_dict[ 'repositories' ]: + exclude_dict[ 'repositories' ].append( repository_tuple ) + exclude_list.append( exclude_dict ) + return exclude_list + def register_test_result( url, metadata_id, test_results_dict, repository_info_dict, params ): ''' Update the repository metadata tool_test_results and appropriate flags using the API. @@ -566,6 +618,10 @@ def main(): repositories_passed = [] repositories_failed = [] repositories_failed_install = [] + exclude_list = [] + if os.path.exists( exclude_list_file ): + exclude_list = parse_exclude_list( exclude_list_file ) + log.info( exclude_list ) try: # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. log.info( "Retrieving repositories to install from the URL:\n%s\n", str( galaxy_tool_shed_url ) ) @@ -623,8 +679,43 @@ def main(): # Get the name and owner out of the repository info dict. name = str( repository_info_dict[ 'name' ] ) owner = str( repository_info_dict[ 'owner' ] ) - log.info( "Installing and testing revision %s of repository id %s (%s/%s)...", - str( changeset_revision ), str( repository_id ), owner, name ) + # Populate the repository_status dict now. + repository_status = get_tool_test_results_from_api( galaxy_tool_shed_url, metadata_revision_id ) + if 'test_environment' not in repository_status: + repository_status[ 'test_environment' ] = {} + test_environment = get_test_environment( repository_status[ 'test_environment' ] ) + test_environment[ 'galaxy_database_version' ] = get_database_version( app ) + test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) + repository_status[ 'test_environment' ] = test_environment + repository_status[ 'passed_tests' ] = [] + repository_status[ 'failed_tests' ] = [] + repository_status[ 'skip_reason' ] = None + # Iterate through the list of repositories defined not to be installed. This should be a list of dicts in the following format: + # { + # 'reason': The default reason or the reason specified in this section, + # 'repositories': + # [ + # ( name, owner, changeset revision if changeset revision else None ), + # ( name, owner, changeset revision if changeset revision else None ) + # ] + # }, + # If changeset revision is None, that means the entire repository is excluded from testing, otherwise only the specified + # revision should be skipped. + # TODO: When a repository is selected to be skipped, use the API to update the tool shed with the defined skip reason. + skip_this_repository = False + skip_because = None + for exclude_by_reason in exclude_list: + reason = exclude_by_reason[ 'reason' ] + exclude_repositories = exclude_by_reason[ 'repositories' ] + if ( name, owner, changeset_revision ) in exclude_repositories or ( name, owner, None ) in exclude_repositories: + skip_this_repository = True + skip_because = reason + break + if skip_this_repository: + log.info( "Not testing revision %s of repository %s owned by %s.", changeset_revision, name, owner ) + continue + else: + log.info( "Installing and testing revision %s of repository %s owned by %s...", changeset_revision, name, owner ) # Explicitly clear tests from twill's test environment. remove_generated_tests( app ) # Use the repository information dict to generate an install method that will install the repository into the embedded @@ -638,15 +729,6 @@ def main(): # repository, with tool and repository dependencies also selected for installation. result, _ = run_tests( test_config ) success = result.wasSuccessful() - repository_status = get_tool_test_results_from_api( galaxy_tool_shed_url, metadata_revision_id ) - if 'test_environment' not in repository_status: - repository_status[ 'test_environment' ] = {} - test_environment = get_test_environment( repository_status[ 'test_environment' ] ) - test_environment[ 'galaxy_database_version' ] = get_database_version( app ) - test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) - repository_status[ 'test_environment' ] = test_environment - repository_status[ 'passed_tests' ] = [] - repository_status[ 'failed_tests' ] = [] repository_status[ 'installation_errors' ] = dict( current_repository=[], repository_dependencies=[], tool_dependencies=[] ) try: repository = test_db_util.get_installed_repository_by_name_owner_changeset_revision( name, owner, changeset_revision ) @@ -750,16 +832,19 @@ def main(): # If the repository does not have a test-data directory, any functional tests in the tool configuration will # fail. Mark the repository as failed and skip installation. log.error( 'Test data is missing for this repository. Updating repository and skipping functional tests.' ) - # Record the lack of test data. - for tool in repository.metadata[ 'tools' ]: - tool_id = tool[ 'id' ] - tool_version = tool[ 'version' ] - tool_guid = tool[ 'guid' ] - # In keeping with the standard display layout, add the error message to the dict for each tool individually. - missing_components = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, - missing_components="Repository %s is missing a test-data directory." % name ) - if missing_components not in repository_status[ 'missing_test_components' ]: - repository_status[ 'missing_test_components' ].append( missing_components ) + # Record the lack of test data if the repository metadata defines tools. + if 'tools' in repository.metadata: + for tool in repository.metadata[ 'tools' ]: + tool_id = tool[ 'id' ] + tool_version = tool[ 'version' ] + tool_guid = tool[ 'guid' ] + # In keeping with the standard display layout, add the error message to the dict for each tool individually. + missing_components = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid, + missing_components="Repository %s is missing a test-data directory." % name ) + if missing_components not in repository_status[ 'missing_test_components' ]: + repository_status[ 'missing_test_components' ].append( missing_components ) + else: + continue # Record the status of this repository in the tool shed. set_do_not_test = not is_latest_downloadable_revision( galaxy_tool_shed_url, repository_info_dict ) params[ 'tools_functionally_correct' ] = False From 0618f403654d9344c43db656103b2e908abc2a5f Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 21 May 2013 16:21:17 -0400 Subject: [PATCH 016/318] Fix for detecting Data Manager tool dependencies when installed from a tool shed. --- lib/galaxy/tools/__init__.py | 13 +++++++++++++ lib/galaxy/tools/data_manager/manager.py | 15 ++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index fa4d8198ad4..686a205e7bd 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -952,6 +952,8 @@ class Tool( object ): self.version = None # Enable easy access to this tool's version lineage. self.lineage_ids = [] + #populate toolshed repository info, if available + self.populate_tool_shed_info() # Parse XML element containing configuration self.parse( root, guid=guid ) self.external_runJob_script = app.config.drmaa_external_runjob_script @@ -1775,6 +1777,17 @@ class Tool( object ): version = requirement_elem.get( "version", None ) requirement = ToolRequirement( name=name, type=type, version=version ) self.requirements.append( requirement ) + + def populate_tool_shed_info( self ): + if self.repository_id is not None and 'ToolShedRepository' in self.app.model: + repository_id = self.app.security.decode_id( self.repository_id ) + tool_shed_repository = self.sa_session.query( self.app.model.ToolShedRepository ).get( repository_id ) + if tool_shed_repository: + self.tool_shed = tool_shed_repository.tool_shed + self.repository_name = tool_shed_repository.name + self.repository_owner = tool_shed_repository.owner + self.installed_changeset_revision = tool_shed_repository.installed_changeset_revision + def check_workflow_compatible( self, root ): """ Determine if a tool can be used in workflows. External tools and the diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index b966c3d79ca..acd2f106bb8 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -9,6 +9,7 @@ from galaxy import util from galaxy.util.odict import odict from galaxy.util.template import fill_template from galaxy.tools.data import TabularToolDataTable +import tool_shed.util.shed_util_common as suc #set up logger import logging @@ -113,12 +114,20 @@ class DataManager( object ): self.guid = elem.get( 'guid', None ) path = elem.get( 'tool_file', None ) self.version = elem.get( 'version', self.version ) + tool_shed_repository_id = None tool_guid = None if path is None: tool_elem = elem.find( 'tool' ) assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) ) path = tool_elem.get( "file", None ) tool_guid = tool_elem.get( "guid", None ) + #need to determine repository info so that dependencies will work correctly + tool_shed = tool_elem.find( 'tool_shed' ).text + repository_name = tool_elem.find( 'repository_name' ).text + repository_owner = tool_elem.find( 'repository_owner' ).text + installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text + tool_shed_repository = suc.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision( self.data_managers.app, tool_shed, repository_name, repository_owner, installed_changeset_revision ) + tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id ) #use shed_conf_file to determine tool_path shed_conf_file = elem.get( "shed_conf_file", None ) if shed_conf_file: @@ -126,7 +135,7 @@ class DataManager( object ): if shed_conf: tool_path = shed_conf.get( "tool_path", tool_path ) assert path is not None, "A tool file path could not be determined:\n%s" % ( util.xml_to_string( elem ) ) - self.load_tool( os.path.join( tool_path, path ), guid=tool_guid, data_manager_id=self.id ) + self.load_tool( os.path.join( tool_path, path ), guid=tool_guid, data_manager_id=self.id, tool_shed_repository_id=tool_shed_repository_id ) self.name = elem.get( 'name', self.tool.name ) self.description = elem.get( 'description', self.tool.description ) @@ -180,8 +189,8 @@ class DataManager( object ): @property def id( self ): return self.guid or self.declared_id #if we have a guid, we will use that as the data_manager id - def load_tool( self, tool_filename, guid=None, data_manager_id=None ): - tool = self.data_managers.app.toolbox.load_tool( tool_filename, guid=guid, data_manager_id=data_manager_id ) + def load_tool( self, tool_filename, guid=None, data_manager_id=None, tool_shed_repository_id=None ): + tool = self.data_managers.app.toolbox.load_tool( tool_filename, guid=guid, data_manager_id=data_manager_id, repository_id=tool_shed_repository_id ) self.data_managers.app.toolbox.data_manager_tools[ tool.id ] = tool self.data_managers.app.toolbox.tools_by_id[ tool.id ] = tool self.tool = tool From ab80d7713e648a4ce13c213befbe592bedb1f00c Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Wed, 22 May 2013 09:20:31 -0400 Subject: [PATCH 017/318] Error handling and logging for the install and test exclude list. --- .../functional_tests.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/install_and_test_tool_shed_repositories/functional_tests.py b/test/install_and_test_tool_shed_repositories/functional_tests.py index 7ca68b0aa8c..a96240b5924 100644 --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -368,24 +368,34 @@ def parse_exclude_list( xml_filename ): ] ''' exclude_list = [] + exclude_verbose = [] xml_tree = parse_xml( xml_filename ) tool_sheds = xml_tree.findall( 'repositories' ) - xml_element = None + xml_element = [] + exclude_count = 0 for tool_shed in tool_sheds: if galaxy_tool_shed_url != tool_shed.attrib[ 'tool_shed' ]: continue else: xml_element = tool_shed for reason_section in xml_element: - print reason_section reason_text = reason_section.find( 'text' ).text repositories = reason_section.findall( 'repository' ) exclude_dict = dict( reason=reason_text, repositories=[] ) for repository in repositories: repository_tuple = get_repository_tuple_from_elem( repository ) if repository_tuple not in exclude_dict[ 'repositories' ]: + exclude_verbose.append( repository_tuple ) + exclude_count += 1 exclude_dict[ 'repositories' ].append( repository_tuple ) exclude_list.append( exclude_dict ) + log.debug( '%d repositories excluded from testing...', exclude_count ) + if '-list_repositories' in sys.argv: + for name, owner, changeset_revision in exclude_verbose: + if changeset_revision: + log.debug( 'Repository %s owned by %s, changeset revision %s.', name, owner, changeset_revision ) + else: + log.debug( 'Repository %s owned by %s, all revisions.', name, owner ) return exclude_list def register_test_result( url, metadata_id, test_results_dict, repository_info_dict, params ): @@ -620,8 +630,8 @@ def main(): repositories_failed_install = [] exclude_list = [] if os.path.exists( exclude_list_file ): + log.info( 'Loading the list of repositories excluded from testing from the file %s...', exclude_list_file ) exclude_list = parse_exclude_list( exclude_list_file ) - log.info( exclude_list ) try: # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. log.info( "Retrieving repositories to install from the URL:\n%s\n", str( galaxy_tool_shed_url ) ) From edaf91fc58bb2eec9bdc64362f63ab94b7f928b1 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 22 May 2013 09:51:47 -0400 Subject: [PATCH 018/318] Fix for RawObjectWrapper when None values are provided. --- lib/galaxy/tools/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 686a205e7bd..1b6a2bab78b 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3264,7 +3264,11 @@ class RawObjectWrapper( ToolParameterValueWrapper ): def __nonzero__( self ): return bool( self.obj ) #FIXME: would it be safe/backwards compatible to rename .obj to .value, so that we can just inherit this method? def __str__( self ): - return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) + try: + return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__) + except: + #Most likely None, which lacks __module__. + return str( self.obj ) def __getattr__( self, key ): return getattr( self.obj, key ) From 76e4979c1d11c9095078e0263dceeffcd65a5f69 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Wed, 22 May 2013 09:52:03 -0400 Subject: [PATCH 019/318] Eliminate links in the tool shed menu when they are not necessary. --- templates/webapps/tool_shed/index.mako | 87 +++++++++++++------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/templates/webapps/tool_shed/index.mako b/templates/webapps/tool_shed/index.mako index 615f603a0a9..4985ee04daa 100644 --- a/templates/webapps/tool_shed/index.mako +++ b/templates/webapps/tool_shed/index.mako @@ -92,51 +92,54 @@ Browse by category %if trans.user: -
-
- Repositories I Can Change -
-
- Repositories I own -
- %if has_reviewed_repositories: + %if trans.user.active_repositories: +
+
+ Repositories I Can Change +
- Reviewed repositories I own + Repositories I own +
+ %if has_reviewed_repositories: +
+ Reviewed repositories I own +
+ %endif + %if has_deprecated_repositories: +
+ Deprecated repositories I own +
+ %endif +
+ My writable repositories +
+
+ Latest revision: missing tool tests +
+
+ Latest revision: installation errors +
+
+ Latest revision: failing tool tests +
+
+ Latest revision: skip tool tests +
+
+ Latest revision: all tool tests pass +
+
+ Latest revision: invalid tools +
+ %else: +
+
+ Available Actions +
+
+ Create new repository
%endif - %if has_deprecated_repositories: -
- Deprecated repositories I own -
- %endif -
- My writable repositories -
-
- Latest revision: missing tool tests -
-
- Latest revision: installation errors -
-
- Latest revision: failing tool tests -
-
- Latest revision: skip tool tests -
-
- Latest revision: all tool tests pass -
-
- Latest revision: invalid tools -
-
-
- Available Actions -
-
- Create new repository -
%if can_review_repositories:
From a787646f925b41437759f1926507ced6741e9909 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Wed, 22 May 2013 14:36:08 -0400 Subject: [PATCH 020/318] Delete the original copy of the install_repository_tools.py Galaxy API script since it has been renamed to install_tool_shed_repositories.py --- scripts/api/install_repository_tools.py | 52 ------------------------- 1 file changed, 52 deletions(-) delete mode 100644 scripts/api/install_repository_tools.py diff --git a/scripts/api/install_repository_tools.py b/scripts/api/install_repository_tools.py deleted file mode 100644 index 508b8236ce1..00000000000 --- a/scripts/api/install_repository_tools.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python -""" -Install a specified repository revision from a specified tool shed into Galaxy. This example demonstrates installation of a repository that contains -valid tools, loading them into a section of the Galaxy tool panel or creating a new tool panel section. -You can choose if tool dependencies or repository dependencies should be installed, use --repository-deps or --tool-deps. - -This example requires a tool panel config file (e.g., tool_conf.xml, shed_tool_conf.xml, etc) to contain a tool panel section like the following: - -
-
- -Here is a working example of how to use this script to install a repository from the test tool shed. -./install_repository_tools.py --api --local --url http://testtoolshed.g2.bx.psu.edu --name gregs_filter --owner greg --revision f28d5018f9cb --tool-deps -""" - -import os -import sys -import argparse -sys.path.insert( 0, os.path.dirname( __file__ ) ) -from common import submit - -def main( options ): - """Collect all user data and install the tools via the Galaxy API.""" - data = {} - data[ 'tool_shed_url' ] = options.tool_shed_url - data[ 'name' ] = options.name - data[ 'owner' ] = options.owner - data[ 'changeset_revision' ] = options.changeset_revision - if options.tool_panel_section_id: - data[ 'tool_panel_section_id' ] = options.tool_panel_section_id - elif options.new_tool_panel_section_label: - data[ 'new_tool_panel_section_label' ] = options.new_tool_panel_section_label - if options.install_repository_dependencies: - data[ 'install_repository_dependencies' ] = options.install_repository_dependencies - if options.install_tool_dependencies: - data[ 'install_tool_dependencies' ] = options.install_tool_dependencies - submit( options.api, '%s%s' % ( options.local_url.strip('/'), '/api/tool_shed_repositories/new/install_repository_revision' ), data ) - -if __name__ == '__main__': - parser = argparse.ArgumentParser( description='Installation of tool shed repositories via the Galaxy API.' ) - parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) - parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) - parser.add_argument( "-l", "--local", dest="local_url", required=True, help="URL of the galaxy instance." ) - parser.add_argument( "-n", "--name", required=True, help="Repository name." ) - parser.add_argument( "-o", "--owner", required=True, help="Repository owner." ) - parser.add_argument( "-r", "--revision", dest="changeset_revision", required=True, help="Repository owner." ) - parser.add_argument( "--panel-section-id", dest="tool_panel_section_id", help="Tool panel section id if you want to add your repository to an existing tool section." ) - parser.add_argument( "--panel-section-name", dest="new_tool_panel_section_label", help="New tool panel section label. If specified a new tool section will be created." ) - parser.add_argument( "--repository-deps", dest="install_repository_dependencies", action="store_true", default=False, help="Install repository dependencies. [False]") - parser.add_argument( "--tool-deps", dest="install_tool_dependencies", action="store_true", default=False, help="Install tool dependencies. [False]" ) - options = parser.parse_args() - main( options ) From 839fe5dc9d9fcfccd116c794dd14093ff6022804 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Wed, 22 May 2013 15:28:40 -0400 Subject: [PATCH 021/318] Fix recent changes to the tool shed menu to allow a logged in user to create a new repository. --- templates/webapps/tool_shed/index.mako | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/templates/webapps/tool_shed/index.mako b/templates/webapps/tool_shed/index.mako index 4985ee04daa..4dec1012da1 100644 --- a/templates/webapps/tool_shed/index.mako +++ b/templates/webapps/tool_shed/index.mako @@ -131,15 +131,14 @@
Latest revision: invalid tools
- %else: -
-
- Available Actions -
-
- Create new repository -
%endif +
+
+ Available Actions +
+
+ Create new repository +
%if can_review_repositories:
From fd6f41e375bf7adab583fd4ce7152682cdf749f5 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Wed, 22 May 2013 16:08:38 -0400 Subject: [PATCH 022/318] Eliminate the use of util.Params from the tool shed's repository controller. --- .../tool_shed/controllers/repository.py | 218 ++++++++---------- 1 file changed, 95 insertions(+), 123 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 14aeeacad0a..21f769223e7 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -657,10 +657,9 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def browse_repository( self, trans, id, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - commit_message = util.restore_text( params.get( 'commit_message', 'Deleted selected files' ) ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + commit_message = kwd.get( 'commit_message', 'Deleted selected files' ) repository = suc.get_repository_in_tool_shed( trans, id ) repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) # Update repository files for browsing. @@ -833,14 +832,13 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def check_for_updates( self, trans, **kwd ): """Handle a request from a local Galaxy instance.""" - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) # If the request originated with the UpdateManager, it will not include a galaxy_url. galaxy_url = suc.handle_galaxy_url( trans, **kwd ) - name = params.get( 'name', None ) - owner = params.get( 'owner', None ) - changeset_revision = params.get( 'changeset_revision', None ) + name = kwd.get( 'name', None ) + owner = kwd.get( 'owner', None ) + changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) @@ -913,9 +911,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def contact_owner( self, trans, id, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, id ) metadata = self.get_metadata( trans, id, repository.tip( trans.app ) ) if trans.user and trans.user.email: @@ -948,9 +945,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def create_repository( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) categories = suc.get_categories( trans ) if not categories: message = 'No categories have been configured in this instance of the Galaxy Tool Shed. ' + \ @@ -960,12 +956,12 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): action='browse_repositories', message=message, status=status ) ) - name = util.restore_text( params.get( 'name', '' ) ) - description = util.restore_text( params.get( 'description', '' ) ) - long_description = util.restore_text( params.get( 'long_description', '' ) ) - category_ids = util.listify( params.get( 'category_id', '' ) ) + name = kwd.get( 'name', '' ) + description = kwd.get( 'description', '' ) + long_description = kwd.get( 'long_description', '' ) + category_ids = util.listify( kwd.get( 'category_id', '' ) ) selected_categories = [ trans.security.decode_id( id ) for id in category_ids ] - if params.get( 'create_repository_button', False ): + if kwd.get( 'create_repository_button', False ): error = False message = self.__validate_repository_name( name, trans.user ) if message: @@ -1031,12 +1027,11 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): """Mark a repository in the tool shed as deprecated or not deprecated.""" # Marking a repository in the tool shed as deprecated has no effect on any downloadable changeset revisions that may be associated with the # repository. Revisions are not marked as not downlaodable because those that have installed the repository must be allowed to get updates. - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - repository_id = params.get( 'id', None ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + repository_id = kwd.get( 'id', None ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) - mark_deprecated = util.string_as_bool( params.get( 'mark_deprecated', False ) ) + mark_deprecated = util.string_as_bool( kwd.get( 'mark_deprecated', False ) ) repository.deprecated = mark_deprecated trans.sa_session.add( repository ) trans.sa_session.flush() @@ -1052,9 +1047,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def display_tool( self, trans, repository_id, tool_config, changeset_revision, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) repository, tool, message = tool_util.load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config ) if message: status = 'error' @@ -1105,7 +1099,6 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def download( self, trans, repository_id, changeset_revision, file_type, **kwd ): # Download an archive of the repository files compressed as zip, gz or bz2. - params = util.Params( kwd ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) # Allow hgweb to handle the download. This requires the tool shed # server account's .hgrc file to include the following setting: @@ -1125,9 +1118,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def find_tools( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) galaxy_url = suc.handle_galaxy_url( trans, **kwd ) if 'operation' in kwd: item_id = kwd.get( 'id', '' ) @@ -1169,7 +1161,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): tool_ids = [ item.lower() for item in util.listify( kwd.get( 'tool_id', '' ) ) ] tool_names = [ item.lower() for item in util.listify( kwd.get( 'tool_name', '' ) ) ] tool_versions = [ item.lower() for item in util.listify( kwd.get( 'tool_version', '' ) ) ] - exact_matches = params.get( 'exact_matches', '' ) + exact_matches = kwd.get( 'exact_matches', '' ) exact_matches_checked = CheckboxField.is_checked( exact_matches ) match_tuples = [] ok = True @@ -1210,9 +1202,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def find_workflows( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) galaxy_url = suc.handle_galaxy_url( trans, **kwd ) if 'operation' in kwd: item_id = kwd.get( 'id', '' ) @@ -1253,7 +1244,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): return trans.show_error_message( "No items were selected." ) if 'find_workflows_button' in kwd: workflow_names = [ item.lower() for item in util.listify( kwd.get( 'workflow_name', '' ) ) ] - exact_matches = params.get( 'exact_matches', '' ) + exact_matches = kwd.get( 'exact_matches', '' ) exact_matches_checked = CheckboxField.is_checked( exact_matches ) match_tuples = [] ok = True @@ -1327,12 +1318,11 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): if 'workflows' in metadata: includes_workflows = True return includes_data_managers, includes_datatypes, includes_tools, includes_tools_for_display_in_tool_panel, includes_tool_dependencies, has_repository_dependencies, includes_workflows - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - name = params.get( 'name', None ) - owner = params.get( 'owner', None ) - changeset_revision = params.get( 'changeset_revision', None ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + name = kwd.get( 'name', None ) + owner = kwd.get( 'owner', None ) + changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, trans.security.encode_id( repository.id ), @@ -1518,10 +1508,9 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.json def get_repository_dependencies( self, trans, **kwd ): """Return an encoded dictionary of all repositories upon which the contents of the received repository depends.""" - params = util.Params( kwd ) - name = params.get( 'name', None ) - owner = params.get( 'owner', None ) - changeset_revision = params.get( 'changeset_revision', None ) + name = kwd.get( 'name', None ) + owner = kwd.get( 'owner', None ) + changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) repository_id = trans.security.encode_id( repository.id ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) @@ -1615,10 +1604,9 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def get_tool_dependencies( self, trans, **kwd ): """Handle a request from a Galaxy instance to get the tool_dependencies entry from the metadata for a specified changeset revision.""" - params = util.Params( kwd ) - name = params.get( 'name', None ) - owner = params.get( 'owner', None ) - changeset_revision = params.get( 'changeset_revision', None ) + name = kwd.get( 'name', None ) + owner = kwd.get( 'owner', None ) + changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) for downloadable_revision in repository.downloadable_revisions: if downloadable_revision.changeset_revision == changeset_revision: @@ -1632,10 +1620,9 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def get_tool_dependencies_config_contents( self, trans, **kwd ): """Handle a request from a Galaxy instance to get the tool_dependencies.xml file contents for a specified changeset revision.""" - params = util.Params( kwd ) - name = params.get( 'name', None ) - owner = params.get( 'owner', None ) - changeset_revision = params.get( 'changeset_revision', None ) + name = kwd.get( 'name', None ) + owner = kwd.get( 'owner', None ) + changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) # TODO: We're currently returning the tool_dependencies.xml file that is available on disk. We need to enhance this process # to retrieve older versions of the tool-dependencies.xml file from the repository manafest. @@ -1772,9 +1759,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def help( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = util.restore_text( kwd.get( 'message', '' ) ) + status = kwd.get( 'status', 'done' ) return trans.fill_template( '/webapps/tool_shed/repository/help.mako', message=message, status=status, **kwd ) def __in_tool_dict( self, tool_dict, exact_matches_checked, tool_id=None, tool_name=None, tool_version=None ): @@ -1826,9 +1812,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def index( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) # See if there are any RepositoryMetadata records since menu items require them. repository_metadata = trans.sa_session.query( model.RepositoryMetadata ).first() current_user = trans.user @@ -1847,9 +1832,9 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): break # Route in may have been from a sharable URL, in whcih case we'll have a user_id and possibly a name # The received user_id will be the id of the repository owner. - user_id = params.get( 'user_id', None ) - repository_id = params.get( 'repository_id', None ) - changeset_revision = params.get( 'changeset_revision', None ) + user_id = kwd.get( 'user_id', None ) + repository_id = kwd.get( 'repository_id', None ) + changeset_revision = kwd.get( 'changeset_revision', None ) return trans.fill_template( '/webapps/tool_shed/index.mako', repository_metadata=repository_metadata, has_reviewed_repositories=has_reviewed_repositories, @@ -1890,9 +1875,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def load_invalid_tool( self, trans, repository_id, tool_config, changeset_revision, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'error' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'error' ) repository, tool, error_message = tool_util.load_tool_from_changeset_revision( trans, repository_id, changeset_revision, tool_config ) tool_state = self.__new_state( trans ) invalid_file_tups = [] @@ -1944,13 +1928,12 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose @web.require_login( "manage email alerts" ) def manage_email_alerts( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - new_repo_alert = params.get( 'new_repo_alert', '' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + new_repo_alert = kwd.get( 'new_repo_alert', '' ) new_repo_alert_checked = CheckboxField.is_checked( new_repo_alert ) user = trans.user - if params.get( 'new_repo_alert_button', False ): + if kwd.get( 'new_repo_alert_button', False ): user.new_repo_alert = new_repo_alert_checked trans.sa_session.add( user ) trans.sa_session.flush() @@ -2209,9 +2192,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose @web.require_login( "multi select email alerts" ) def multi_select_email_alerts( self, trans, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) if 'operation' in kwd: operation = kwd[ 'operation' ].lower() if operation == "receive email alerts": @@ -2249,13 +2231,12 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def preview_tools_in_changeset( self, trans, repository_id, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) - changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip( trans.app ) ) ) + changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) if repository_metadata: repository_metadata_id = trans.security.encode_id( repository_metadata.id ), @@ -2302,12 +2283,11 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): older changeset_revsion, but later the repository was updated in the tool shed and the Galaxy admin is trying to install the latest changeset revision of the same repository instead of updating the one that was previously installed. """ - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - name = params.get( 'name', None ) - owner = params.get( 'owner', None ) - changeset_revision = params.get( 'changeset_revision', None ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + name = kwd.get( 'name', None ) + owner = kwd.get( 'owner', None ) + changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) @@ -2326,10 +2306,9 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.require_login( "rate repositories" ) def rate_repository( self, trans, **kwd ): """ Rate a repository and return updated rating data. """ - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - id = params.get( 'id', None ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + id = kwd.get( 'id', None ) if not id: return trans.response.send_redirect( web.url_for( controller='repository', action='browse_repositories', @@ -2342,12 +2321,12 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): action='browse_repositories', message="You are not allowed to rate your own repository", status='error' ) ) - if params.get( 'rate_button', False ): - rating = int( params.get( 'rating', '0' ) ) - comment = util.restore_text( params.get( 'comment', '' ) ) + if kwd.get( 'rate_button', False ): + rating = int( kwd.get( 'rating', '0' ) ) + comment = kwd.get( 'comment', '' ) rating = self.rate_item( trans, trans.user, repository, rating, comment ) avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, repository, webapp_model=trans.model ) - display_reviews = util.string_as_bool( params.get( 'display_reviews', False ) ) + display_reviews = util.string_as_bool( kwd.get( 'display_reviews', False ) ) rra = self.get_user_item_rating( trans.sa_session, trans.user, repository, webapp_model=trans.model ) metadata = self.get_metadata( trans, id, repository.tip( trans.app ) ) return trans.fill_template( '/webapps/tool_shed/repository/rate_repository.mako', @@ -2480,15 +2459,14 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def select_files_to_delete( self, trans, id, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) - commit_message = util.restore_text( params.get( 'commit_message', 'Deleted selected files' ) ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) + commit_message = kwd.get( 'commit_message', 'Deleted selected files' ) repository = suc.get_repository_in_tool_shed( trans, id ) repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) - selected_files_to_delete = util.restore_text( params.get( 'selected_files_to_delete', '' ) ) - if params.get( 'select_files_to_delete_button', False ): + selected_files_to_delete = kwd.get( 'selected_files_to_delete', '' ) + if kwd.get( 'select_files_to_delete_button', False ): if selected_files_to_delete: selected_files_to_delete = selected_files_to_delete.split( ',' ) # Get the current repository tip. @@ -2744,10 +2722,9 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): revision for the required repository, making it impossible to discover the repository without knowledge of revisions to which it could have been updated. """ - params = util.Params( kwd ) - name = params.get( 'name', None ) - owner = params.get( 'owner', None ) - changeset_revision = params.get( 'changeset_revision', None ) + name = kwd.get( 'name', None ) + owner = kwd.get( 'owner', None ) + changeset_revision = kwd.get( 'changeset_revision', None ) repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) repo_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_dir ) @@ -2784,9 +2761,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def view_changelog( self, trans, id, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, id ) repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) changesets = [] @@ -2820,9 +2796,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def view_changeset( self, trans, id, ctx_str, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, id ) repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) ctx = suc.get_changectx_for_changeset( repo, ctx_str ) @@ -2898,15 +2873,14 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def view_repository( self, trans, id, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, id ) repo = hg.repository( suc.get_configured_ui(), repository.repo_path( trans.app ) ) avg_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, repository, webapp_model=trans.model ) - changeset_revision = util.restore_text( params.get( 'changeset_revision', repository.tip( trans.app ) ) ) - display_reviews = util.string_as_bool( params.get( 'display_reviews', False ) ) - alerts = params.get( 'alerts', '' ) + changeset_revision = kwd.get( 'changeset_revision', repository.tip( trans.app ) ) + display_reviews = util.string_as_bool( kwd.get( 'display_reviews', False ) ) + alerts = kwd.get( 'alerts', '' ) alerts_checked = CheckboxField.is_checked( alerts ) if repository.email_alerts: email_alerts = json.from_json_string( repository.email_alerts ) @@ -2914,7 +2888,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): email_alerts = [] repository_dependencies = None user = trans.user - if user and params.get( 'receive_email_alerts_button', False ): + if user and kwd.get( 'receive_email_alerts_button', False ): flush_needed = False if alerts_checked: if user.email not in email_alerts: @@ -2982,9 +2956,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def view_tool_metadata( self, trans, repository_id, changeset_revision, tool_id, **kwd ): - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) repository = suc.get_repository_in_tool_shed( trans, repository_id ) repo_files_dir = repository.repo_path( trans.app ) repo = hg.repository( suc.get_configured_ui(), repo_files_dir ) @@ -3054,9 +3027,8 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): @web.expose def view_workflow( self, trans, workflow_name, repository_metadata_id, **kwd ): """Retrieve necessary information about a workflow from the database so that it can be displayed in an svg image.""" - params = util.Params( kwd ) - message = util.restore_text( params.get( 'message', '' ) ) - status = params.get( 'status', 'done' ) + message = kwd.get( 'message', '' ) + status = kwd.get( 'status', 'done' ) if workflow_name: workflow_name = encoding_util.tool_shed_decode( workflow_name ) repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) From 21f70fb6fdfb06071c693ac00c5501d4f9622958 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Wed, 22 May 2013 16:09:52 -0400 Subject: [PATCH 023/318] White space fixes in the tool shed registry code. --- lib/tool_shed/tool_shed_registry.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/tool_shed/tool_shed_registry.py b/lib/tool_shed/tool_shed_registry.py index bc06af41251..26a5aaa3d54 100644 --- a/lib/tool_shed/tool_shed_registry.py +++ b/lib/tool_shed/tool_shed_registry.py @@ -11,7 +11,7 @@ log = logging.getLogger( __name__ ) class Registry( object ): def __init__( self, root_dir=None, config=None ): - self.tool_sheds = odict() + self.tool_sheds = odict() self.tool_sheds_auth = odict() if root_dir and config: # Parse tool_sheds_conf.xml @@ -20,17 +20,17 @@ class Registry( object ): log.debug( 'Loading references to tool sheds from %s' % config ) for elem in root.findall( 'tool_shed' ): try: - name = elem.get( 'name', None ) - url = elem.get( 'url', None ) + name = elem.get( 'name', None ) + url = elem.get( 'url', None ) username = elem.get( 'user', None ) password = elem.get( 'pass', None ) if name and url: - self.tool_sheds[ name ] = url + self.tool_sheds[ name ] = url self.tool_sheds_auth[ name ] = None log.debug( 'Loaded reference to tool shed: %s' % name ) if name and url and username and password: pass_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() - pass_mgr.add_password(None, url, username, password) + pass_mgr.add_password( None, url, username, password ) self.tool_sheds_auth[ name ] = pass_mgr except Exception, e: log.warning( 'Error loading reference to tool shed "%s", problem: %s' % ( name, str( e ) ) ) From ebcf49083d4894283d99a14c38db6a92138bec7d Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Thu, 23 May 2013 11:11:37 -0400 Subject: [PATCH 024/318] Fix for TabularToolDataTable.get_column_name_list() when value column is overloaded by e.g. name. --- lib/galaxy/tools/data/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index bb079c607ef..1cd2b95ba81 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -297,9 +297,12 @@ class TabularToolDataTable( ToolDataTable ): found_column = False for name, index in self.columns.iteritems(): if index == i: - rval.append( name ) + if not found_column: + rval.append( name ) + elif name == 'value': + #the column named 'value' always has priority over other named columns + rval[ -1 ] = name found_column = True - break if not found_column: rval.append( None ) return rval From 8a15acb8a8d2cffa3c0435f5c20bcdaf2fa09ec8 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Thu, 23 May 2013 11:11:37 -0400 Subject: [PATCH 025/318] Fix for TabularToolDataTable.get_column_name_list() when value column is overloaded by e.g. name. --- lib/galaxy/tools/data/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 92103823898..7f5938a51ec 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -273,9 +273,12 @@ class TabularToolDataTable( ToolDataTable ): found_column = False for name, index in self.columns.iteritems(): if index == i: - rval.append( name ) + if not found_column: + rval.append( name ) + elif name == 'value': + #the column named 'value' always has priority over other named columns + rval[ -1 ] = name found_column = True - break if not found_column: rval.append( None ) return rval From 9179fb57554eaefc9f50b30b21dcf0ed6bcddb5c Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Fri, 24 May 2013 11:26:19 -0400 Subject: [PATCH 026/318] Enhance the install and test framework to support testing a single changeset revision of a repository. This defaults to testing all repositories and changeset revisions determined to be installable and testable. --- .../functional_tests.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/install_and_test_tool_shed_repositories/functional_tests.py b/test/install_and_test_tool_shed_repositories/functional_tests.py index a96240b5924..f8297261848 100644 --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -145,7 +145,15 @@ if 'GALAXY_INSTALL_TEST_SECRET' not in os.environ: else: galaxy_encode_secret = os.environ[ 'GALAXY_INSTALL_TEST_SECRET' ] - +testing_single_repository = {} +if 'repository_name' in os.environ and 'repository_owner' in os.environ: + testing_single_repository[ 'name' ] = os.environ[ 'repository_name' ] + testing_single_repository[ 'owner' ] = os.environ[ 'repository_owner' ] + if 'repository_revision' in os.environ: + testing_single_repository[ 'changeset_revision' ] = os.environ[ 'repository_revision' ] + else: + testing_single_repository[ 'changeset_revision' ] = None + class ReportResults( Plugin ): '''Simple Nose plugin to record the IDs of all tests run, regardless of success.''' name = "reportresults" @@ -308,6 +316,19 @@ def get_repositories_to_install( tool_shed_url, latest_revision_only=True ): skipped_previous = ' and metadata revisions that are not the most recent' else: skipped_previous = '' + if testing_single_repository: + log.info( 'Testing single repository with name %s and owner %s.', + testing_single_repository[ 'name' ], + testing_single_repository[ 'owner' ]) + for repository_to_install in detailed_repository_list: + if repository_to_install[ 'name' ] == testing_single_repository[ 'name' ] \ + and repository_to_install[ 'owner' ] == testing_single_repository[ 'owner' ]: + if testing_single_repository[ 'changeset_revision' ] is None: + return [ repository_to_install ] + else: + if testing_single_repository[ 'changeset_revision' ] == repository_to_install[ 'changeset_revision' ]: + return [ repository_to_install ] + return [] log.info( 'After removing deleted repositories%s from the list, %d remain to be tested.', skipped_previous, repositories_tested ) return detailed_repository_list From 35515ca218c715f75cbcee43c79e8ccff573b1c8 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 24 May 2013 13:04:32 -0400 Subject: [PATCH 027/318] Fix for MetadataInDataTableColumnValidator becoming stale when a tool data table is updated during Galaxy runtime. --- lib/galaxy/tools/data/__init__.py | 75 ++++++++++++++++++++++- lib/galaxy/tools/data_manager/manager.py | 44 +------------ lib/galaxy/tools/parameters/validation.py | 19 +++++- 3 files changed, 92 insertions(+), 46 deletions(-) diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index 1cd2b95ba81..0431e15be1c 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -161,10 +161,23 @@ class ToolDataTable( object ): self.tool_data_file = None self.tool_data_path = tool_data_path self.missing_index_file = None + # increment this variable any time a new entry is added, or when the table is totally reloaded + # This value has no external meaning, and does not represent an abstract version of the underlying data + self._loaded_content_version = 1 def get_empty_field_by_name( self, name ): return self.empty_field_values.get( name, self.empty_field_value ) - + + def _add_entry( self, entry, persist=False, persist_on_error=False, **kwd ): + raise NotImplementedError( "Abstract method" ) + + def add_entry( self, entry, persist=False, persist_on_error=False, **kwd ): + self._add_entry( entry, persist=persist, persist_on_error=persist_on_error, **kwd ) + self._loaded_content_version += 1 + return self._loaded_content_version + + def is_current_version( self, other_version ): + return self._loaded_content_version == other_version class TabularToolDataTable( ToolDataTable ): """ @@ -234,6 +247,9 @@ class TabularToolDataTable( ToolDataTable ): def get_fields( self ): return self.data + + def get_version_fields( self ): + return ( self._loaded_content_version, self.data ) def parse_column_spec( self, config_element ): """ @@ -324,6 +340,61 @@ class TabularToolDataTable( ToolDataTable ): rval = fields[ return_col ] break return rval - + + def _add_entry( self, entry, persist=False, persist_on_error=False, **kwd ): + #accepts dict or list of columns + if isinstance( entry, dict ): + fields = [] + for column_name in self.get_column_name_list(): + if column_name not in entry: + log.debug( "Using default column value for column '%s' when adding data table entry (%s) to table '%s'.", column_name, entry, self.name ) + field_value = self.get_empty_field_by_name( column_name ) + else: + field_value = entry[ column_name ] + fields.append( field_value ) + else: + fields = entry + if self.largest_index < len( fields ): + fields = self._replace_field_separators( fields ) + self.data.append( fields ) + field_len_error = False + else: + log.error( "Attempted to add fields (%s) to data table '%s', but there were not enough fields specified ( %i < %i ).", fields, self.name, len( fields ), self.largest_index + 1 ) + field_len_error = True + if persist and ( not field_len_error or persist_on_error ): + #FIXME: Need to lock these files for editing + try: + data_table_fh = open( self.filename, 'r+b' ) + except IOError, e: + log.warning( 'Error opening data table file (%s) with r+b, assuming file does not exist and will open as wb: %s', self.filename, e ) + data_table_fh = open( self.filename, 'wb' ) + if os.stat( self.filename )[6] != 0: + # ensure last existing line ends with new line + data_table_fh.seek( -1, 2 ) #last char in file + last_char = data_table_fh.read( 1 ) + if last_char not in [ '\n', '\r' ]: + data_table_fh.write( '\n' ) + data_table_fh.write( "%s\n" % ( self.separator.join( fields ) ) ) + return not field_len_error + + def _replace_field_separators( self, fields, separator=None, replace=None, comment_char=None ): + #make sure none of the fields contain separator + #make sure separator replace is different from comment_char, + #due to possible leading replace + if separator is None: + separator = self.separator + if replace is None: + if separator == " ": + if comment_char == "\t": + replace = "_" + else: + replace = "\t" + else: + if comment_char == " ": + replace = "_" + else: + replace = " " + return map( lambda x: x.replace( separator, replace ), fields ) + # Registry of tool data types by type_key tool_data_table_types = dict( [ ( cls.type_key, cls ) for cls in [ TabularToolDataTable ] ] ) diff --git a/lib/galaxy/tools/data_manager/manager.py b/lib/galaxy/tools/data_manager/manager.py index acd2f106bb8..18261735d6b 100644 --- a/lib/galaxy/tools/data_manager/manager.py +++ b/lib/galaxy/tools/data_manager/manager.py @@ -233,59 +233,21 @@ class DataManager( object ): assert output_ref_dataset is not None, "Referenced output was not found." output_ref_values[ data_table_column ] = output_ref_dataset - final_data_table_values = [] if not isinstance( data_table_values, list ): data_table_values = [ data_table_values ] - columns = data_table.get_column_name_list() - #FIXME: Need to lock these files for editing - try: - data_table_fh = open( data_table.filename, 'r+b' ) - except IOError, e: - log.warning( 'Error opening data table file (%s) with r+b, assuming file does not exist and will open as wb: %s' % ( data_table.filename, e ) ) - data_table_fh = open( data_table.filename, 'wb' ) - if os.stat( data_table.filename )[6] != 0: - # ensure last existing line ends with new line - data_table_fh.seek( -1, 2 ) #last char in file - last_char = data_table_fh.read() - if last_char not in [ '\n', '\r' ]: - data_table_fh.write( '\n' ) for data_table_row in data_table_values: data_table_value = dict( **data_table_row ) #keep original values here for name, value in data_table_row.iteritems(): #FIXME: need to loop through here based upon order listed in data_manager config if name in output_ref_values: moved = self.process_move( data_table_name, name, output_ref_values[ name ].extra_files_path, **data_table_value ) data_table_value[ name ] = self.process_value_translation( data_table_name, name, **data_table_value ) - final_data_table_values.append( data_table_value ) - fields = [] - for column_name in columns: - if column_name is None or column_name not in data_table_value: - fields.append( data_table.get_empty_field_by_name( column_name ) ) - else: - fields.append( data_table_value[ column_name ] ) - #should we add a comment to file about automatically generated value here? - data_table_fh.write( "%s\n" % ( data_table.separator.join( self._replace_field_separators( fields, separator=data_table.separator ) ) ) ) #write out fields to disk - data_table.data.append( fields ) #add fields to loaded data table - data_table_fh.close() + data_table.add_entry( data_table_value, persist=True ) + for data_table_name, data_table_values in data_tables_dict.iteritems(): #tool returned extra data table entries, but data table was not declared in data manager #do not add these values, but do provide messages log.warning( 'The data manager "%s" returned an undeclared data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( self.id, data_table_name, data_table_values, data_table_name, self.data_managers.filename ) ) - def _replace_field_separators( self, fields, separator="\t", replace=None, comment_char=None ): - #make sure none of the fields contain separator - #make sure separator replace is different from comment_char, - #due to possible leading replace - if replace is None: - if separator == " ": - if comment_char == "\t": - replace = "_" - else: - replace = "\t" - else: - if comment_char == " ": - replace = "_" - else: - replace = " " - return map( lambda x: x.replace( separator, replace ), fields ) + def process_move( self, data_table_name, column_name, source_base_path, relative_symlinks=False, **kwd ): if data_table_name in self.move_by_data_table_column and column_name in self.move_by_data_table_column[ data_table_name ]: move_dict = self.move_by_data_table_column[ data_table_name ][ column_name ] diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index af71b282128..85acd5b1cd4 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -293,18 +293,31 @@ class MetadataInDataTableColumnValidator( Validator ): if line_startswith: line_startswith = line_startswith.strip() return cls( tool_data_table, metadata_name, metadata_column, message, line_startswith ) + def __init__( self, tool_data_table, metadata_name, metadata_column, message="Value for metadata not found.", line_startswith=None ): self.metadata_name = metadata_name self.message = message self.valid_values = [] + self._data_table_content_version = None + self._tool_data_table = tool_data_table if isinstance( metadata_column, basestring ): metadata_column = tool_data_table.columns[ metadata_column ] - for fields in tool_data_table.get_fields(): - if metadata_column < len( fields ): - self.valid_values.append( fields[ metadata_column ] ) + self._metadata_column = metadata_column + self._load_values() + + def _load_values( self ): + self._data_table_content_version, data_fields = self._tool_data_table.get_version_fields() + self.valid_values = [] + for fields in data_fields: + if self._metadata_column < len( fields ): + self.valid_values.append( fields[ self._metadata_column ] ) + def validate( self, value, history = None ): if not value: return if hasattr( value, "metadata" ): + if not self._tool_data_table.is_current_version( self._data_table_content_version ): + log.debug( 'MetadataInDataTableColumnValidator values are out of sync with data table (%s), updating validator.', self._tool_data_table.name ) + self._load_values() if value.metadata.spec[self.metadata_name].param.to_string( value.metadata.get( self.metadata_name ) ) in self.valid_values: return raise ValueError( self.message ) From 7deb34ea821eff93bde12b1d9f2026b7aefde955 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 24 May 2013 13:36:17 -0400 Subject: [PATCH 028/318] Fix for displaying unicode characters in select menus in form_builder. --- lib/galaxy/web/form_builder.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index a153140094b..d594995f23a 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -351,11 +351,17 @@ class SelectField(BaseField): if selected: selected_text = " selected" last_selected_value = value + if not isinstance( last_selected_value, basestring ): + last_selected_value = str( last_selected_value ) else: selected_text = "" - rval.append( '' % ( escape( str( value ), quote=True ), selected_text, escape( str( text ), quote=True ) ) ) + if not isinstance( value, basestring ): + value = str( value ) + if not isinstance( text, basestring ): + text = str( text ) + rval.append( '' % ( escape( unicodify( value ), quote=True ), selected_text, escape( unicodify( text ), quote=True ) ) ) if last_selected_value: - last_selected_value = ' last_selected_value="%s"' % escape( str( last_selected_value ), quote=True ) + last_selected_value = ' last_selected_value="%s"' % escape( unicodify( last_selected_value ), quote=True ) rval.insert( 0, '' ) From 05c4d6db0f7cf9a63949f4036849ca0e973c5879 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 24 May 2013 13:43:25 -0400 Subject: [PATCH 029/318] Fix for displaying unicode characters in select checkboxes in form_builder. --- lib/galaxy/web/form_builder.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index d594995f23a..c8b1a7edc25 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -300,7 +300,12 @@ class SelectField(BaseField): rval.append ( '
' % ( prefix, self.name ) ) #placeholder for the insertion of the Select All/Unselect All buttons for text, value, selected in self.options: style = "" - escaped_value = escape( str( value ), quote=True ) + if not isinstance( value, basestring ): + value = str( value ) + if not isinstance( text, basestring ): + text = str( text ) + text = unicodify( text ) + escaped_value = escape( unicodify( value ), quote=True ) uniq_id = "%s%s|%s" % (prefix, self.name, escaped_value) if len(self.options) > 2 and ctr % 2 == 1: style = " class=\"odd_row\"" @@ -308,7 +313,7 @@ class SelectField(BaseField): if selected: selected_text = " checked='checked'" rval.append( '
' % \ - ( style, prefix, self.name, escaped_value, uniq_id, selected_text, self.get_disabled_str( disabled ), uniq_id, escape( str( text ), quote=True ) ) ) + ( style, prefix, self.name, escaped_value, uniq_id, selected_text, self.get_disabled_str( disabled ), uniq_id, escape( text, quote=True ) ) ) ctr += 1 return unicodify( "\n".join( rval ) ) def get_html_radio( self, prefix="", disabled=False ): From eb4787eb0e41cb75519453caf3751737f0e6646b Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 24 May 2013 13:56:42 -0400 Subject: [PATCH 030/318] Fix for displaying unicode characters in TextField in form_builder. --- lib/galaxy/web/form_builder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/web/form_builder.py b/lib/galaxy/web/form_builder.py index c8b1a7edc25..e199aca68ee 100644 --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -34,8 +34,12 @@ class TextField(BaseField): self.size = int( size or 10 ) self.value = value or "" def get_html( self, prefix="", disabled=False ): + value = self.value + if not isinstance( value, basestring ): + value = str( value ) + value = unicodify( value ) return unicodify( '' \ - % ( prefix, self.name, self.size, escape( str( self.value ), quote=True ), self.get_disabled_str( disabled ) ) ) + % ( prefix, self.name, self.size, escape( value, quote=True ), self.get_disabled_str( disabled ) ) ) def set_size(self, size): self.size = int( size ) From a89228d5df3e430cffb705ddfa176032c1064255 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 24 May 2013 13:59:26 -0400 Subject: [PATCH 031/318] Unicode fixes for tool parameters --- lib/galaxy/tools/parameters/basic.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 35ace7dee92..1f8dacba407 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -7,7 +7,7 @@ from elementtree.ElementTree import XML, Element from galaxy import config, datatypes, util from galaxy.web import form_builder from galaxy.util.bunch import Bunch -from galaxy.util import string_as_bool, sanitize_param +from galaxy.util import string_as_bool, sanitize_param, unicodify from sanitize import ToolParameterSanitizer import validation, dynamic_options # For BaseURLToolParameter @@ -112,7 +112,9 @@ class ToolParameter( object ): def to_string( self, value, app ): """Convert a value to a string representation suitable for persisting""" - return str( value ) + if not isinstance( value, basestring ): + value = str( value ) + return unicodify( value ) def to_python( self, value, app ): """Convert a value created with to_string back to an object representation""" @@ -144,13 +146,13 @@ class ToolParameter( object ): Convert a value to a text representation suitable for displaying to the user """ - return value + return unicodify( value ) def to_param_dict_string( self, value, other_values={} ): """Called via __str__ when used in the Cheetah template""" if value is None: value = "" - else: + elif not isinstance( value, basestring ): value = str( value ) if self.tool is None or self.tool.options.sanitize: if self.sanitizer: From 7c4c69fd3ecdef8857dd4f817e6c8ae126905155 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Fri, 24 May 2013 15:14:08 -0400 Subject: [PATCH 032/318] Add some important documentation to some tool shed methods. --- .../webapps/tool_shed/util/container_util.py | 13 ++-- .../tool_dependencies/fabric_util.py | 68 +++++++++++++++++++ 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/util/container_util.py b/lib/galaxy/webapps/tool_shed/util/container_util.py index 59a04350e58..e1f85a03ec9 100644 --- a/lib/galaxy/webapps/tool_shed/util/container_util.py +++ b/lib/galaxy/webapps/tool_shed/util/container_util.py @@ -798,12 +798,13 @@ def build_repository_containers_for_tool_shed( trans, repository, changeset_revi label='Valid tools' ) containers_dict[ 'valid_tools' ] = valid_tools_root_folder # Tool test results container. - if tool_test_results: - # Only create and populate this folder if there are actual tool test results to display, since the - # display of the 'Test environment' folder by itself can be misleading. - if len( tool_test_results ) > 1: - folder_id, tool_test_results_root_folder = build_tool_test_results_folder( trans, folder_id, tool_test_results, time_last_tested=time_last_tested ) - containers_dict[ 'tool_test_results' ] = tool_test_results_root_folder + if tool_test_results and len( tool_test_results ) > 1: + # Only create and populate this folder if there are actual tool test results to display, since the display of the 'Test environment' + # folder by itself can be misleading. We check for more than a single entry in the tool_test_results dictionary because it may have + # only the "test_environment" entry, but we want at least 1 of "passed_tests", "failed_tests", "installation_errors", "missing_test_components" + # "skipped_tests", "not_tested" or any other entry that may be added in the future. + folder_id, tool_test_results_root_folder = build_tool_test_results_folder( trans, folder_id, tool_test_results, time_last_tested=time_last_tested ) + containers_dict[ 'tool_test_results' ] = tool_test_results_root_folder # Workflows container. if metadata: if 'workflows' in metadata: diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py index 82f09e573c6..7c63f47a784 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -51,6 +51,70 @@ def handle_command( app, tool_dependency, install_dir, cmd, return_output=False return output.return_code def handle_environment_variables( app, tool_dependency, install_dir, env_var_dict, set_prior_environment_commands ): + """ + This method works with with a combination of three tool dependency definition tag sets, which are defined in the tool_dependencies.xml file in the + order discussed here. The example for this discussion is the tool_dependencies.xml file contained in the osra repository, which is available at: + + http://testtoolshed.g2.bx.psu.edu/view/bgruening/osra + + The first tag set defines a complex repository dependency like this. This tag set ensures that changeset revision XXX of the repository named + package_graphicsmagick_1_3 owned by YYY in the tool shed ZZZ has been previously installed. + + + + + + ... + + * By the way, there is an env.sh file associated with version 1.3.18 of the graphicsmagick package which looks something like this (we'll reference + this file later in this discussion. + ---- + GRAPHICSMAGICK_ROOT_DIR=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick; + export GRAPHICSMAGICK_ROOT_DIR + ---- + + The second tag set defines a specific package dependency that has been previously installed (guaranteed by the tag set discussed above) and compiled, + where the compiled dependency is needed by the tool dependency currently being installed (osra version 2.0.0 in this case) and complied in order for + it's installation and compilation to succeed. This tag set is contained within the tag set, which implies that + version 2.0.0 of the osra package requires version 1.3.18 of the graphicsmagick package in order to successfully compile. When this tag set is handled, + one of the effects is that the env.sh file associated with graphicsmagick version 1.3.18 is "sourced", which undoubtedly sets or alters certain environment + variables (e.g. PATH, PYTHONPATH, etc). + + + + + + + + + The third tag set enables discovery of the same required package dependency discussed above for correctly compiling the osra version 2.0.0 package, but + in this case the package can be discovered at tool execution time. Using the $ENV[] option as shown in this example, the value of the environment + variable named GRAPHICSMAGICK_ROOT_DIR (which was set in the environment using the second tag set described above) will be used to automatically alter + the env.sh file associated with the osra version 2.0.0 tool dependency when it is installed into Galaxy. * Refer to where we discussed the env.sh file + for version 1.3.18 of the graphicsmagick package above. + + + $ENV[$GRAPHICSMAGICK_ROOT_DIR]/lib/ + $INSTALL_DIR/potrace/build/lib/ + $INSTALL_DIR/bin + + $INSTALL_DIR/share + + + The above tag will produce an env.sh file for version 2.0.0 of the osra package when it it installed into Galaxy that looks something like this. Notice + that the path to the gmagick binary is included here since it expands the defined $ENV[$GRAPHICSMAGICK_ROOT_DIR] value in the above tag set. + + ---- + LD_LIBRARY_PATH=//graphicsmagick/1.3.18/YYY/package_graphicsmagick_1_3/XXX/gmagick/lib/:$LD_LIBRARY_PATH; + export LD_LIBRARY_PATH + LD_LIBRARY_PATH=//osra/1.4.0/YYY/depends_on/XXX/potrace/build/lib/:$LD_LIBRARY_PATH; + export LD_LIBRARY_PATH + PATH=//osra/1.4.0/YYY/depends_on/XXX/bin:$PATH; + export PATH + OSRA_DATA_FILES=//osra/1.4.0/YYY/depends_on/XXX/share; + export OSRA_DATA_FILES + ---- + """ env_var_value = env_var_dict[ 'value' ] if '$ENV[' in env_var_value and ']' in env_var_value: # Pull out the name of the environment variable to populate. @@ -107,6 +171,8 @@ def install_and_build_package( app, tool_dependency, actions_dict ): filtered_actions = actions[ 1: ] url = action_dict[ 'url' ] if 'target_filename' in action_dict: + # Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this + # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction. downloaded_filename = action_dict[ 'target_filename' ] else: downloaded_filename = os.path.split( url )[ -1 ] @@ -137,6 +203,8 @@ def install_and_build_package( app, tool_dependency, actions_dict ): filtered_actions = actions[ 1: ] url = action_dict[ 'url' ] if action_dict[ 'target_filename' ]: + # Sometimes compressed archives extracts their content to a folder other than the default defined file name. Using this + # attribute will ensure that the file name is set appropriately and can be located after download, decompression and extraction. filename = action_dict[ 'target_filename' ] else: filename = url.split( '/' )[ -1 ] From 196256f93deb2aa272817c8ec6079fa6c74fb437 Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Sun, 26 May 2013 14:05:31 -0500 Subject: [PATCH 033/318] Trackster: whitespace additions between functions. --- static/scripts/viz/trackster/tracks.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/static/scripts/viz/trackster/tracks.js b/static/scripts/viz/trackster/tracks.js index 1507e5b7a54..8a1b4375478 100644 --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -1137,9 +1137,11 @@ var TracksterView = Backbone.View.extend({ // FIXME: need to use this approach to enable inheritance of DrawableCollection functions. extend( TracksterView.prototype, DrawableCollection.prototype, { + changed: function() { this.has_changes = true; }, + /** Add or remove intro div depending on view state. */ update_intro_div: function() { if (this.drawables.length === 0) { @@ -1149,6 +1151,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { this.intro_div.hide(); } }, + /** * Triggers navigate events as needed. If there is a delay, * then event is triggered only after navigation has stopped. @@ -1171,6 +1174,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { view.trigger("navigate", new_chrom + ":" + new_low + "-" + new_high); } }, + update_location: function(low, high) { this.location_span.text( commatize(low) + ' - ' + commatize(high) ); this.nav_input.val( this.chrom + ':' + commatize(low) + '-' + commatize(high) ); @@ -1182,6 +1186,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { this.trigger_navigate(chrom, view.low, view.high, true); } }, + /** * Load chrom data for the view. Returns a jQuery Deferred. */ @@ -1230,6 +1235,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { return chrom_data; }, + change_chrom: function(chrom, low, high) { var view = this; // If chrom data is still loading, wait for it. @@ -1365,6 +1371,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { var chrom = view.chrom_select.val(); this.trigger_navigate(chrom, view.low, view.high, true); }, + /** * Add a drawable to the view. */ @@ -1374,11 +1381,13 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { this.changed(); this.update_intro_div(); }, + add_label_track: function (label_track) { label_track.view = this; label_track.init(); this.label_tracks.push(label_track); }, + /** * Remove drawable from the view. */ @@ -1392,11 +1401,13 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { }); } }, + reset: function() { this.low = this.max_low; this.high = this.max_high; this.viewport_container.find(".yaxislabel").remove(); }, + /** * Request that view redraw some or all tracks. If a track is not specificied, redraw all tracks. */ @@ -1439,6 +1450,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { this.requested_redraw = true; } }, + /** * Redraws view and tracks. * NOTE: this method should never be called directly; request_redraw() should be used so @@ -1503,6 +1515,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { this.label_tracks[i]._draw(); } }, + zoom_in: function (point, container) { if (this.max_high === 0 || this.high - this.low <= this.min_separation) { return; @@ -1518,6 +1531,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { this.changed(); this.request_redraw(); }, + zoom_out: function () { if (this.max_high === 0) { return; @@ -1530,10 +1544,12 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { this.changed(); this.request_redraw(); }, + resize_window: function() { this.viewport_container.height( this.container.height() - this.top_container.height() - this.bottom_container.height() ); this.request_redraw(); }, + /** Show a Drawable in the overview. */ set_overview: function(drawable) { if (this.overview_drawable) { @@ -1561,6 +1577,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { view.overview_drawable.request_draw(); this.changed(); }, + view /** Close and reset overview. */ reset_overview: function() { // Update UI. From cce3fac45765a75f4b478203f6bfaf2697d29972 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 26 May 2013 15:10:43 -0500 Subject: [PATCH 034/318] Fix API uploads (must have broken with API/Web transaction unification). --- lib/galaxy/tools/actions/upload.py | 4 +-- lib/galaxy/tools/actions/upload_common.py | 36 +++++++++++++++-------- lib/galaxy/webapps/galaxy/api/tools.py | 1 - 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index 85c66fddcfb..4f3f988f676 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -16,7 +16,7 @@ class UploadToolAction( ToolAction ): incoming = upload_common.persist_uploads( incoming ) # We can pass an empty string as the cntrller here since it is used to check whether we # are in an admin view, and this tool is currently not used there. - uploaded_datasets = upload_common.get_uploaded_datasets( trans, '', incoming, precreated_datasets, dataset_upload_inputs ) + uploaded_datasets = upload_common.get_uploaded_datasets( trans, '', incoming, precreated_datasets, dataset_upload_inputs, history=history ) upload_common.cleanup_unused_precreated_datasets( precreated_datasets ) if not uploaded_datasets: @@ -24,4 +24,4 @@ class UploadToolAction( ToolAction ): json_file_path = upload_common.create_paramfile( trans, uploaded_datasets ) data_list = [ ud.data for ud in uploaded_datasets ] - return upload_common.create_job( trans, incoming, tool, json_file_path, data_list ) + return upload_common.create_job( trans, incoming, tool, json_file_path, data_list, history=history ) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index a533d7cb35b..98bc5ac33bd 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -107,11 +107,13 @@ def cleanup_unused_precreated_datasets( precreated_datasets ): data.state = data.states.ERROR data.info = 'No file contents were available.' -def new_history_upload( trans, uploaded_dataset, state=None ): +def __new_history_upload( trans, uploaded_dataset, history=None, state=None ): + if not history: + history = trans.history hda = trans.app.model.HistoryDatasetAssociation( name = uploaded_dataset.name, extension = uploaded_dataset.file_type, dbkey = uploaded_dataset.dbkey, - history = trans.history, + history = history, create_dataset = True, sa_session = trans.sa_session ) if state: @@ -120,12 +122,13 @@ def new_history_upload( trans, uploaded_dataset, state=None ): hda.state = hda.states.QUEUED trans.sa_session.add( hda ) trans.sa_session.flush() - trans.history.add_dataset( hda, genome_build = uploaded_dataset.dbkey ) - permissions = trans.app.security_agent.history_get_default_permissions( trans.history ) + history.add_dataset( hda, genome_build=uploaded_dataset.dbkey ) + permissions = trans.app.security_agent.history_get_default_permissions( history ) trans.app.security_agent.set_all_dataset_permissions( hda.dataset, permissions ) trans.sa_session.flush() return hda -def new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state=None ): + +def __new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state=None ): current_user_roles = trans.get_current_user_roles() if not ( ( trans.user_is_admin() and cntrller in [ 'library_admin', 'api' ] ) or trans.app.security_agent.can_add_library_item( current_user_roles, library_bunch.folder ) ): # This doesn't have to be pretty - the only time this should happen is if someone's being malicious. @@ -206,19 +209,22 @@ def new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state= trans.sa_session.add( dp ) trans.sa_session.flush() return ldda -def new_upload( trans, cntrller, uploaded_dataset, library_bunch=None, state=None ): +def new_upload( trans, cntrller, uploaded_dataset, library_bunch=None, history=None, state=None ): if library_bunch: - return new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state ) + return __new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state ) + elif history: + return __new_history_upload( trans, uploaded_dataset, history=history, state=state ) else: - return new_history_upload( trans, uploaded_dataset, state ) -def get_uploaded_datasets( trans, cntrller, params, precreated_datasets, dataset_upload_inputs, library_bunch=None ): + raise Exception("new_upload must be called with empty values for library_bunch and history") + +def get_uploaded_datasets( trans, cntrller, params, precreated_datasets, dataset_upload_inputs, library_bunch=None, history=None ): uploaded_datasets = [] for dataset_upload_input in dataset_upload_inputs: uploaded_datasets.extend( dataset_upload_input.get_uploaded_datasets( trans, params ) ) for uploaded_dataset in uploaded_datasets: data = get_precreated_dataset( precreated_datasets, uploaded_dataset.name ) if not data: - data = new_upload( trans, cntrller, uploaded_dataset, library_bunch ) + data = new_upload( trans, cntrller, uploaded_dataset, library_bunch=library_bunch, history=history ) else: data.extension = uploaded_dataset.file_type data.dbkey = uploaded_dataset.dbkey @@ -246,7 +252,9 @@ def get_uploaded_datasets( trans, cntrller, params, precreated_datasets, dataset trans.sa_session.add( info_association ) trans.sa_session.flush() else: - trans.history.genome_build = uploaded_dataset.dbkey + if not history: + history = trans.history + history.genome_build = uploaded_dataset.dbkey uploaded_dataset.data = data return uploaded_datasets def create_paramfile( trans, uploaded_datasets ): @@ -320,7 +328,7 @@ def create_paramfile( trans, uploaded_datasets ): if trans.app.config.external_chown_script: _chown( json_file_path ) return json_file_path -def create_job( trans, params, tool, json_file_path, data_list, folder=None ): +def create_job( trans, params, tool, json_file_path, data_list, folder=None, history=None ): """ Create the upload job. """ @@ -333,7 +341,9 @@ def create_job( trans, params, tool, json_file_path, data_list, folder=None ): if folder: job.library_folder_id = folder.id else: - job.history_id = trans.history.id + if not history: + history = trans.history + job.history_id = history.id job.tool_id = tool.id job.tool_version = tool.version job.state = job.states.UPLOAD diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 3559fc2784b..97646ed4ca4 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -78,7 +78,6 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): if history_id: target_history = trans.sa_session.query(trans.app.model.History).get( trans.security.decode_id(history_id)) - trans.galaxy_session.current_history = target_history else: target_history = None From 01e803561daed2e4fdb03bb20e68ad710c3b4957 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 26 May 2013 15:15:19 -0500 Subject: [PATCH 035/318] Fix job splitting to rewrite references in config files in addition to command-line. --- lib/galaxy/jobs/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 6e032e20875..4e96f23de07 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1573,6 +1573,12 @@ class TaskWrapper(JobWrapper): self.sa_session.flush() # Build any required config files config_filenames = self.tool.build_config_files( param_dict, self.working_directory ) + for config_filename in config_filenames: + config_contents = open(config_filename, "r").read() + for k, v in fnames.iteritems(): + config_contents = config_contents.replace(k, v) + open(config_filename, "w").write(config_contents) + # FIXME: Build the param file (might return None, DEPRECATED) param_filename = self.tool.build_param_file( param_dict, self.working_directory ) # Build the job's command line From 8126d3f0fa6ea0d94085ea8af83b3662fe642e0d Mon Sep 17 00:00:00 2001 From: Gert Hulselmans Date: Mon, 27 May 2013 15:02:06 +0200 Subject: [PATCH 036/318] Fix detection of 2bit files when uploading them. When 2bit files are uploaded, they were not detected as 2bit files: Example: test.2bit empty format: twobit, database: hg19 The uploaded binary file contains inappropriate content This issue is solved by registering the TwoBit file format in: ./lib/galaxy/datatypes/binary.py --- lib/galaxy/datatypes/binary.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 2a478d7f6df..0bd799db6e2 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -399,3 +399,5 @@ class TwoBit (Binary): return dataset.peek except: return "Binary TwoBit format nucleotide file (%s)" % (data.nice_size(dataset.get_size())) + +Binary.register_sniffable_binary_format("twobit", "twobit", TwoBit) From b3e6055cb9823dd2caeaec9170e8652afd6e4d4e Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Mon, 27 May 2013 13:03:58 -0700 Subject: [PATCH 037/318] Starting the provenance API. Provides information about the source of datasets. --- lib/galaxy/webapps/galaxy/api/provenance.py | 92 +++++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 10 +++ 2 files changed, 102 insertions(+) create mode 100644 lib/galaxy/webapps/galaxy/api/provenance.py diff --git a/lib/galaxy/webapps/galaxy/api/provenance.py b/lib/galaxy/webapps/galaxy/api/provenance.py new file mode 100644 index 00000000000..9b0f8e02fe2 --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/provenance.py @@ -0,0 +1,92 @@ +""" +API operations provenance +""" +import logging +from galaxy import web +from galaxy.web.base.controller import BaseAPIController +from paste.httpexceptions import HTTPNotImplemented, HTTPBadRequest + +log = logging.getLogger( __name__ ) + + +class BaseProvenanceController( BaseAPIController ): + """ + """ + @web.expose_api + def index( self, trans, **kwd ): + value = self._get_provenance( trans, self.provenance_item_class, kwd[self.provenance_item_id] ) + return value + + @web.expose_api + def show( self, trans, elem_name, **kwd ): + value = self._get_provenance( trans, self.provenance_item_class, kwd[self.provenance_item_id] ) + return value + + @web.expose_api + def create( self, trans, tag_name, payload={}, **kwd ): + raise HTTPNotImplemented() + + @web.expose_api + def delete( self, trans, tag_name, **kwd ): + raise HTTPBadRequest("Cannot Delete Provenance") + + def _get_provenance(self, trans, item_class_name, item_id, follow=True): + out = [] + provenance_item = self.get_object( trans, item_id, item_class_name, check_ownership=False, check_accessible=True ) + out.append( + { + "dataset" : { + "id" : trans.security.encode_id(provenance_item.id), + "uuid" : provenance_item.dataset.uuid + }, + "tool_id" : provenance_item.creating_job.tool_id, + "parameters" : self._get_job_record(trans, provenance_item.creating_job) + } + ) + if follow: + depends = self._get_dataset_depends(provenance_item) + recorded = {} + for rec in depends: + if rec.id not in recorded: + recorded[rec.id] = True + out.append( + { + "dataset" : { + "id" : trans.security.encode_id(rec.id), + "uuid" : rec.dataset.uuid + }, + "tool_id" : rec.creating_job.tool_id, + "parameters" : self._get_job_record(trans, rec.creating_job) + } + ) + return out + + def _get_dataset_depends(self, item): + out = [] + for d in item.creating_job.input_datasets: + out.append(d.dataset) + for c in self._get_dataset_depends(d.dataset): + out.append(c) + return out + + def _get_job_record(self, trans, job): + out = {} + for p in job.parameters: + out[p.name] = p.value + for in_d in job.input_datasets: + out[in_d.name] = { "dataset_id" : trans.security.encode_id(in_d.dataset.id), "uuid" : in_d.dataset.dataset.uuid } + return out + + +class HDAProvenanceController( BaseProvenanceController ): + controller_name = "history_content_provenance" + provenance_item_class = "HistoryDatasetAssociation" + provenance_item_id = "history_content_id" + + +class LDDAProvenanceController( BaseProvenanceController ): + controller_name = "ldda_provenance" + provenance_item_class = "LibraryDatasetDatasetAssociation" + provenance_item_id = "library_content_id" + + diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 5123182dcd3..d7041829f0b 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -128,6 +128,10 @@ def app_factory( global_conf, **kwargs ): name_prefix="workflow_", path_prefix='/api/workflows/:workflow_id' ) + _add_item_provenance_controller( webapp, + name_prefix="history_content_", + path_prefix='/api/histories/:history_id/contents/:history_content_id' ) + webapp.mapper.resource( 'dataset', 'datasets', path_prefix='/api' ) webapp.mapper.resource_with_deleted( 'library', 'libraries', path_prefix='/api' ) webapp.mapper.resource( 'sample', 'samples', path_prefix='/api' ) @@ -234,6 +238,12 @@ def _add_item_annotation_controller( webapp, name_prefix, path_prefix, **kwd ): name = "%sannotation" % name_prefix webapp.mapper.resource(name, "annotation", path_prefix=path_prefix, controller=controller) +def _add_item_provenance_controller( webapp, name_prefix, path_prefix, **kwd ): + controller = "%sprovenance" % name_prefix + name = "%sprovenance" % name_prefix + webapp.mapper.resource(name, "provenance", path_prefix=path_prefix, controller=controller) + + def wrap_in_middleware( app, global_conf, **local_conf ): """ Based on the configuration wrap `app` in a set of common and useful From 58261982f5514eec1b9a353b199728fa1fb2a69d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 27 May 2013 21:40:36 -0500 Subject: [PATCH 038/318] Improved exception and error handling conditions for the LWR job runner. PEP-8 fixes. --- lib/galaxy/jobs/runners/lwr.py | 46 ++++++++++++-------- lib/galaxy/jobs/runners/lwr_client/client.py | 11 ++++- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/jobs/runners/lwr.py b/lib/galaxy/jobs/runners/lwr.py index ae7dffbf03d..70edc5d1ae0 100644 --- a/lib/galaxy/jobs/runners/lwr.py +++ b/lib/galaxy/jobs/runners/lwr.py @@ -14,6 +14,7 @@ log = logging.getLogger( __name__ ) __all__ = [ 'LwrJobRunner' ] + class LwrJobRunner( AsynchronousJobRunner ): """ LWR Job Runner @@ -49,18 +50,17 @@ class LwrJobRunner( AsynchronousJobRunner ): return job_state def queue_job(self, job_wrapper): - stderr = stdout = command_line = '' - + command_line = '' job_destination = job_wrapper.job_destination try: job_wrapper.prepare() if hasattr(job_wrapper, 'prepare_input_files_cmds') and job_wrapper.prepare_input_files_cmds is not None: - for cmd in job_wrapper.prepare_input_files_cmds: # run the commands to stage the input files + for cmd in job_wrapper.prepare_input_files_cmds: # run the commands to stage the input files #log.debug( 'executing: %s' % cmd ) if 0 != os.system(cmd): raise Exception('Error running file staging command: %s' % cmd) - job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line + job_wrapper.prepare_input_files_cmds = None # prevent them from being used in-line command_line = self.build_command_line( job_wrapper, include_metadata=False, include_work_dir_outputs=False ) except: job_wrapper.fail( "failure preparing job", exception=True ) @@ -73,7 +73,6 @@ class LwrJobRunner( AsynchronousJobRunner ): return try: - #log.debug( 'executing: %s' % command_line ) client = self.get_client_from_wrapper(job_wrapper) output_files = self.get_output_files(job_wrapper) input_files = job_wrapper.get_input_fnames() @@ -85,8 +84,7 @@ class LwrJobRunner( AsynchronousJobRunner ): client.launch( rebuilt_command_line ) job_wrapper.set_job_destination( job_destination, job_id ) job_wrapper.change_state( model.Job.states.QUEUED ) - - except Exception, exc: + except: job_wrapper.fail( "failure running job", exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) return @@ -124,23 +122,36 @@ class LwrJobRunner( AsynchronousJobRunner ): client = self.get_client_from_state(job_state) run_results = client.raw_check_complete() - log.debug('run_results %s' % run_results ) stdout = run_results['stdout'] stderr = run_results['stderr'] + download_failure_exceptions = [] if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ]: work_dir_outputs = self.get_work_dir_outputs(job_wrapper) output_files = self.get_output_files(job_wrapper) for source_file, output_file in work_dir_outputs: - client.download_work_dir_output(source_file, job_wrapper.working_directory, output_file) + try: + client.download_work_dir_output(source_file, job_wrapper.working_directory, output_file) + except Exception, e: + download_failure_exceptions.append(e) # Remove from full output_files list so don't try to download directly. output_files.remove(output_file) for output_file in output_files: - client.download_output(output_file, working_directory=job_wrapper.working_directory) - client.clean() + try: + client.download_output(output_file, working_directory=job_wrapper.working_directory) + except Exception, e: + download_failure_exceptions.append(e) + if download_failure_exceptions or self.app.config.cleanup_job == "always": + try: + client.clean() + except: + log.warn("Failed to cleanup remote LWR job") + if download_failure_exceptions: + job_wrapper.fail("Failed to find or download one or more job outputs from remote server.", exception=True) log.debug('execution finished: %s' % command_line) - except Exception, exc: - job_wrapper.fail( "failure running job", exception=True ) + except: + message = "Failed to communicate with remote job server." + job_wrapper.fail( message, exception=True ) log.exception("failure running job %d" % job_wrapper.job_id) return self._handle_metadata_externally( job_wrapper ) @@ -166,14 +177,14 @@ class LwrJobRunner( AsynchronousJobRunner ): if e.errno == errno.ESRCH: log.debug( "check_pid(): PID %d is dead" % pid ) else: - log.warning( "check_pid(): Got errno %s when attempting to check PID %d: %s" %( errno.errorcode[e.errno], pid, e.strerror ) ) + log.warning( "check_pid(): Got errno %s when attempting to check PID %d: %s" % ( errno.errorcode[e.errno], pid, e.strerror ) ) return False def stop_job( self, job ): #if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished job_ext_output_metadata = job.get_external_output_metadata() if job_ext_output_metadata: - pid = job_ext_output_metadata[0].job_runner_external_pid #every JobExternalOutputMetadata has a pid set, we just need to take from one of them + pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them if pid in [ None, '' ]: log.warning( "stop_job(): %s: no PID in database for job, unable to stop" % job.id ) return @@ -189,10 +200,10 @@ class LwrJobRunner( AsynchronousJobRunner ): return # give up sleep( 2 ) if not self.check_pid( pid ): - log.debug( "stop_job(): %s: PID %d successfully killed with signal %d" %( job.id, pid, sig ) ) + log.debug( "stop_job(): %s: PID %d successfully killed with signal %d" % ( job.id, pid, sig ) ) return else: - log.warning( "stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" %( job.id, pid ) ) + log.warning( "stop_job(): %s: PID %d refuses to die after signaling TERM/KILL" % ( job.id, pid ) ) else: # Remote kill lwr_url = job.job_runner_name @@ -201,7 +212,6 @@ class LwrJobRunner( AsynchronousJobRunner ): client = self.get_client(job.destination_params, job_id) client.kill() - def recover( self, job, job_wrapper ): """Recovers jobs stuck in the queued/running state when Galaxy started""" job_state = AsynchronousJobState() diff --git a/lib/galaxy/jobs/runners/lwr_client/client.py b/lib/galaxy/jobs/runners/lwr_client/client.py index 9862d35362d..a4e6a3a7d23 100644 --- a/lib/galaxy/jobs/runners/lwr_client/client.py +++ b/lib/galaxy/jobs/runners/lwr_client/client.py @@ -19,6 +19,15 @@ class parseJson(object): return replacement +class OutputNotFoundException(Exception): + + def __init__(self, path): + self.path = path + + def __str__(self): + return "No remote output found for path %s" % self.path + + class Client(object): """ Objects of this client class perform low-level communication with a remote LWR server. @@ -163,7 +172,7 @@ class Client(object): elif output_type == "task": output_path = os.path.join(working_directory, name) else: - raise Exception("No remote output found for dataset with path %s" % path) + raise OutputNotFoundException(path) self.__raw_download_output(name, self.job_id, output_type, output_path) def __raw_download_output(self, name, job_id, output_type, output_path): From 2df81e59de1687dd54b2a9a8896315c34f9d5db6 Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Tue, 28 May 2013 00:05:35 -0700 Subject: [PATCH 039/318] Adding source following in provenance tracing --- lib/galaxy/webapps/galaxy/api/provenance.py | 63 ++++++++------------- 1 file changed, 25 insertions(+), 38 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/provenance.py b/lib/galaxy/webapps/galaxy/api/provenance.py index 9b0f8e02fe2..11b3301d997 100644 --- a/lib/galaxy/webapps/galaxy/api/provenance.py +++ b/lib/galaxy/webapps/galaxy/api/provenance.py @@ -14,12 +14,14 @@ class BaseProvenanceController( BaseAPIController ): """ @web.expose_api def index( self, trans, **kwd ): - value = self._get_provenance( trans, self.provenance_item_class, kwd[self.provenance_item_id] ) + follow = kwd.get('follow', False) + value = self._get_provenance( trans, self.provenance_item_class, kwd[self.provenance_item_id], follow ) return value @web.expose_api def show( self, trans, elem_name, **kwd ): - value = self._get_provenance( trans, self.provenance_item_class, kwd[self.provenance_item_id] ) + follow = kwd.get('follow', False) + value = self._get_provenance( trans, self.provenance_item_class, kwd[self.provenance_item_id], follow ) return value @web.expose_api @@ -31,50 +33,35 @@ class BaseProvenanceController( BaseAPIController ): raise HTTPBadRequest("Cannot Delete Provenance") def _get_provenance(self, trans, item_class_name, item_id, follow=True): - out = [] provenance_item = self.get_object( trans, item_id, item_class_name, check_ownership=False, check_accessible=True ) - out.append( - { - "dataset" : { - "id" : trans.security.encode_id(provenance_item.id), - "uuid" : provenance_item.dataset.uuid - }, - "tool_id" : provenance_item.creating_job.tool_id, - "parameters" : self._get_job_record(trans, provenance_item.creating_job) - } - ) - if follow: - depends = self._get_dataset_depends(provenance_item) - recorded = {} - for rec in depends: - if rec.id not in recorded: - recorded[rec.id] = True - out.append( - { - "dataset" : { - "id" : trans.security.encode_id(rec.id), - "uuid" : rec.dataset.uuid - }, - "tool_id" : rec.creating_job.tool_id, - "parameters" : self._get_job_record(trans, rec.creating_job) - } - ) + out = self._get_record(trans, provenance_item, follow) return out - def _get_dataset_depends(self, item): - out = [] - for d in item.creating_job.input_datasets: - out.append(d.dataset) - for c in self._get_dataset_depends(d.dataset): - out.append(c) - return out + def _get_record(self, trans, item, follow): + if item is not None: + if follow: + if item.copied_from_library_dataset_dataset_association: + item = item.copied_from_library_dataset_dataset_association + return { + "id" : trans.security.encode_id(item.id), + "uuid" : item.dataset.uuid, + "tool_id" : item.creating_job.tool_id, + "parameters" : self._get_job_record(trans, item.creating_job, follow) + } + else: + return { + "id" : trans.security.encode_id(item.id), + "uuid" : item.dataset.uuid + } + return None - def _get_job_record(self, trans, job): + def _get_job_record(self, trans, job, follow): out = {} for p in job.parameters: out[p.name] = p.value for in_d in job.input_datasets: - out[in_d.name] = { "dataset_id" : trans.security.encode_id(in_d.dataset.id), "uuid" : in_d.dataset.dataset.uuid } + if follow: + out[in_d.name] = self._get_record(trans, in_d.dataset, follow) return out From 344c9dcc2ab79e5bd8c92a5fcb852bd63c6ea68d Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Tue, 28 May 2013 10:22:01 -0700 Subject: [PATCH 040/318] Fixing provenance output --- lib/galaxy/webapps/galaxy/api/provenance.py | 27 ++++++++++----------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/provenance.py b/lib/galaxy/webapps/galaxy/api/provenance.py index 11b3301d997..938a4ec702d 100644 --- a/lib/galaxy/webapps/galaxy/api/provenance.py +++ b/lib/galaxy/webapps/galaxy/api/provenance.py @@ -39,20 +39,14 @@ class BaseProvenanceController( BaseAPIController ): def _get_record(self, trans, item, follow): if item is not None: - if follow: - if item.copied_from_library_dataset_dataset_association: - item = item.copied_from_library_dataset_dataset_association - return { - "id" : trans.security.encode_id(item.id), - "uuid" : item.dataset.uuid, - "tool_id" : item.creating_job.tool_id, - "parameters" : self._get_job_record(trans, item.creating_job, follow) - } - else: - return { - "id" : trans.security.encode_id(item.id), - "uuid" : item.dataset.uuid - } + if item.copied_from_library_dataset_dataset_association: + item = item.copied_from_library_dataset_dataset_association + return { + "id" : trans.security.encode_id(item.id), + "uuid" : item.dataset.uuid, + "tool_id" : item.creating_job.tool_id, + "parameters" : self._get_job_record(trans, item.creating_job, follow) + } return None def _get_job_record(self, trans, job, follow): @@ -62,6 +56,11 @@ class BaseProvenanceController( BaseAPIController ): for in_d in job.input_datasets: if follow: out[in_d.name] = self._get_record(trans, in_d.dataset, follow) + else: + out[in_d.name] = { + "id" : trans.security.encode_id(in_d.dataset.id), + "uuid" : in_d.dataset.dataset.uuid + } return out From be5e2777b3723e8a6abad4f3a50ddecbee695c91 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 28 May 2013 17:23:17 -0400 Subject: [PATCH 041/318] Bugfix for standard dataset upload --- lib/galaxy/tools/actions/upload_common.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 98bc5ac33bd..7fe8fa6ce3d 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -209,13 +209,12 @@ def __new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, stat trans.sa_session.add( dp ) trans.sa_session.flush() return ldda + def new_upload( trans, cntrller, uploaded_dataset, library_bunch=None, history=None, state=None ): if library_bunch: return __new_library_upload( trans, cntrller, uploaded_dataset, library_bunch, state ) - elif history: - return __new_history_upload( trans, uploaded_dataset, history=history, state=state ) else: - raise Exception("new_upload must be called with empty values for library_bunch and history") + return __new_history_upload( trans, uploaded_dataset, history=history, state=state ) def get_uploaded_datasets( trans, cntrller, params, precreated_datasets, dataset_upload_inputs, library_bunch=None, history=None ): uploaded_datasets = [] From be2bb18e8330d69c5bf631a1d4a1c3474cc87cb9 Mon Sep 17 00:00:00 2001 From: Carl Eberhard Date: Wed, 29 May 2013 17:28:47 -0400 Subject: [PATCH 042/318] Tools API: better error handling in create, encode ids returned; Scripts: add history_upload script to api --- lib/galaxy/webapps/galaxy/api/tools.py | 18 ++++++--- scripts/api/history_upload.py | 55 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 6 deletions(-) create mode 100755 scripts/api/history_upload.py diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 97646ed4ca4..6240617c212 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -66,9 +66,9 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): # -- Execute tool. -- # Get tool. - tool_id = payload[ 'tool_id' ] - tool = trans.app.toolbox.get_tool( tool_id ) + tool = trans.app.toolbox.get_tool( payload[ 'tool_id' ] ) if 'tool_id' in payload else None if not tool: + trans.response.status = 404 return { "message": { "type": "error", "text" : trans.app.model.Dataset.conversion_messages.NO_TOOL } } # Set running history from payload parameters. @@ -82,7 +82,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): target_history = None # Set up inputs. - inputs = payload[ 'inputs' ] + inputs = payload.get( 'inputs', {} ) # Find files coming in as multipart file data and add to inputs. for k, v in payload.iteritems(): if k.startswith("files_"): @@ -91,17 +91,23 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): # HACK: add run button so that tool.handle_input will run tool. inputs['runtool_btn'] = 'Execute' # TODO: encode data ids and decode ids. + # TODO: handle dbkeys params = util.Params( inputs, sanitize = False ) - template, vars = tool.handle_input( trans, params.__dict__, history=target_history) + template, vars = tool.handle_input( trans, params.__dict__, history=target_history ) + if 'errors' in vars: + trans.response.status = 400 + return { "message": { "type": "error", "data" : vars[ 'errors' ] } } # TODO: check for errors and ensure that output dataset(s) are available. - output_datasets = vars.get('out_data', {}).values() + output_datasets = vars.get( 'out_data', {} ).values() rval = { "outputs": [] } outputs = rval[ "outputs" ] + #TODO:?? poss. only return ids? for output in output_datasets: - outputs.append( output.get_api_value() ) + output_dict = output.get_api_value() + outputs.append( trans.security.encode_dict_ids( output_dict ) ) return rval # diff --git a/scripts/api/history_upload.py b/scripts/api/history_upload.py new file mode 100755 index 00000000000..cf16d5b3fb3 --- /dev/null +++ b/scripts/api/history_upload.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +""" +Upload a file to the desired history. +""" +_USAGE = ( "history_upload.py \n" + + " (where galaxy base url is just the root url where your Galaxy is served; e.g. 'localhost:8080')" ) + +import os, sys, json, pprint +#sys.path.insert( 0, os.path.dirname( __file__ ) ) + +try: + import requests +except ImportError, imp_err: + log.error( "Could not import the requests module. See http://docs.python-requests.org/en/latest/ or " + + "install with 'pip install requests'" ) + raise + +def upload_file( full_url, api_key, history_id, filepath, **kwargs ): + + payload = { + 'key' : api_key, + 'tool_id' : 'upload1', + 'history_id' : history_id, + } + inputs = { + 'files_0|NAME' : kwargs.get( 'filename', os.path.basename( filepath ) ), + 'files_0|type' : 'upload_dataset', + #TODO: the following doesn't work with tools.py + #'dbkey' : kwargs.get( 'dbkey', '?' ), + 'dbkey' : '?', + 'file_type' : kwargs.get( 'file_type', 'auto' ), + 'ajax_upload' : u'true', + 'async_datasets': '1', + } + payload[ 'inputs' ] = json.dumps( inputs ) + + response = None + with open( filepath, 'rb' ) as file_to_upload: + files = { 'files_0|file_data' : file_to_upload } + response = requests.post( full_url, data=payload, files=files ) + return response + +if __name__ == '__main__': + + if len( sys.argv ) < 5: + print _USAGE + sys.exit( 1 ) + + api_key, base_url, history_id, filepath = sys.argv[1:5] + full_url = base_url + '/api/tools' + kwargs = dict([ kwarg.split('=', 1) for kwarg in sys.argv[5:]]) + + response = upload_file( full_url, api_key, history_id, filepath, **kwargs ) + print >> sys.stderr, response + print response.content From f2065f4a24816840c87ac3504fdb3815496b8f43 Mon Sep 17 00:00:00 2001 From: Ross Lazarus Date: Thu, 30 May 2013 14:57:13 +1000 Subject: [PATCH 043/318] Starting the Rgenetics deprecation process - removing all rgenetics sections from tool_conf.xml.sample and fastqc update to dance around EBI_SRA download bogus file extensions --- tool_conf.xml.sample | 32 -------------------------------- tools/rgenetics/rgFastQC.py | 30 +++++++++++++++++++++++++++++- tools/rgenetics/rgFastQC.xml | 2 +- 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/tool_conf.xml.sample b/tool_conf.xml.sample index 69ef9b8fc13..623e71abefd 100644 --- a/tool_conf.xml.sample +++ b/tool_conf.xml.sample @@ -397,38 +397,6 @@
-
- -
-
- -
-
- - - - -
diff --git a/tools/rgenetics/rgFastQC.py b/tools/rgenetics/rgFastQC.py index 3fa071b3a12..9d8933eb5cd 100644 --- a/tools/rgenetics/rgFastQC.py +++ b/tools/rgenetics/rgFastQC.py @@ -1,4 +1,5 @@ """ +# May 2013 ross added check for bogus gz extension - fastqc gets confused # added sanitizer for user supplied name # removed shell and make cl a sequence for Popen call # ross lazarus August 10 2012 in response to anon insecurity report @@ -24,6 +25,8 @@ import optparse import shutil import tempfile from rgutils import getFileString +import zipfile +import gzip class FastQC(): """wrapper @@ -56,7 +59,32 @@ class FastQC(): cl.append('--contaminants=%s' % self.opts.contaminants) # patch suggested by bwlang https://bitbucket.org/galaxy/galaxy-central/pull-request/30 # use a symlink in a temporary directory so that the FastQC report reflects the history input file name - fastqinfilename = re.sub(ur'[^a-zA-Z0-9_\-\.]', '_', os.path.basename(self.opts.inputfilename)) + infname = self.opts.inputfilename + linf = infname.lower() + trimext = False + isgz = linf.endswith('.gz') or linf.endswith('.gzip') + # decompression at upload currently does NOT remove this now bogus ending - fastqc will barf + # patched may 29 2013 until this is fixed properly + if isgz: + f = gzip.open(self.opts.input) + try: + testrow = f.readline() + except: + trimext = True + f.close() + elif linf.endswith('bz2'): + f = bz2.open(self.opts.input,'rb') + try: + f.readline() + except: + trimext = True + f.close() + elif linf.endswith('.zip'): + if not zipfile.is_zipfile(self.opts.input): + trimext = True + if trimext: + infname = os.path.splitext(infname)[0] + fastqinfilename = re.sub(ur'[^a-zA-Z0-9_\-\.]', '_', os.path.basename(infname)) link_name = os.path.join(self.opts.outputdir, fastqinfilename) os.symlink(self.opts.input, link_name) cl.append(link_name) diff --git a/tools/rgenetics/rgFastQC.xml b/tools/rgenetics/rgFastQC.xml index 74ba898d425..528a0740c02 100644 --- a/tools/rgenetics/rgFastQC.xml +++ b/tools/rgenetics/rgFastQC.xml @@ -1,4 +1,4 @@ - + reports using FastQC rgFastQC.py -i "$input_file" -d "$html_file.files_path" -o "$html_file" -n "$out_prefix" -f "$input_file.ext" -j "$input_file.name" -e "${GALAXY_DATA_INDEX_DIR}/shared/jars/FastQC/fastqc" From 941ce7d5ddbb9a949e24d09a01fb8ca75ad9ccca Mon Sep 17 00:00:00 2001 From: Ross Lazarus Date: Thu, 30 May 2013 15:01:29 +1000 Subject: [PATCH 044/318] clean up fastqc updates a tiny bit --- tools/rgenetics/rgFastQC.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/rgenetics/rgFastQC.py b/tools/rgenetics/rgFastQC.py index 9d8933eb5cd..5bfbe0b7dd7 100644 --- a/tools/rgenetics/rgFastQC.py +++ b/tools/rgenetics/rgFastQC.py @@ -62,10 +62,9 @@ class FastQC(): infname = self.opts.inputfilename linf = infname.lower() trimext = False - isgz = linf.endswith('.gz') or linf.endswith('.gzip') # decompression at upload currently does NOT remove this now bogus ending - fastqc will barf # patched may 29 2013 until this is fixed properly - if isgz: + if ( linf.endswith('.gz') or linf.endswith('.gzip') ): f = gzip.open(self.opts.input) try: testrow = f.readline() From 8097da1904836c00d99b9ed476def4532d9f91d0 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 30 May 2013 11:23:29 -0400 Subject: [PATCH 045/318] Add specific ami functionality (via GET param) to cloudlaunch. Prevent existing instance selection when using a share string. --- .../webapps/galaxy/controllers/cloudlaunch.py | 13 ++++++++----- templates/webapps/galaxy/cloud/index.mako | 6 ++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/cloudlaunch.py b/lib/galaxy/webapps/galaxy/controllers/cloudlaunch.py index b5bd250ffee..6c8840e6e64 100644 --- a/lib/galaxy/webapps/galaxy/controllers/cloudlaunch.py +++ b/lib/galaxy/webapps/galaxy/controllers/cloudlaunch.py @@ -35,8 +35,8 @@ class CloudController(BaseUIController): BaseUIController.__init__(self, app) @web.expose - def index(self, trans, share_string=None): - return trans.fill_template("cloud/index.mako", default_keypair = DEFAULT_KEYPAIR, share_string=share_string) + def index(self, trans, share_string=None, ami=None): + return trans.fill_template("cloud/index.mako", default_keypair = DEFAULT_KEYPAIR, share_string=share_string, ami=ami) @web.expose def get_account_info(self, trans, key_id, secret, **kwargs): @@ -92,7 +92,7 @@ class CloudController(BaseUIController): return to_json_string(account_info) @web.expose - def launch_instance(self, trans, cluster_name, password, key_id, secret, instance_type, share_string, keypair, zone=None, **kwargs): + def launch_instance(self, trans, cluster_name, password, key_id, secret, instance_type, share_string, keypair, ami=DEFAULT_AMI, zone=None, **kwargs): ec2_error = None try: # Create security group & key pair used when starting an instance @@ -113,11 +113,14 @@ class CloudController(BaseUIController): user_provided_data['password'] = password if share_string: user_provided_data['share_string'] = share_string + rs = run_instance(ec2_conn=ec2_conn, + image_id = ami, user_provided_data=user_provided_data, key_name=kp_name, security_groups=[sg_name], - placement=zone) + placement=zone + ) if rs: instance = rs.instances[0] ct = 0 @@ -266,7 +269,7 @@ def create_key_pair(ec2_conn, key_name=DEFAULT_KEYPAIR): return None, None return kp.name, kp.material -def run_instance(ec2_conn, user_provided_data, image_id=DEFAULT_AMI, +def run_instance(ec2_conn, user_provided_data, image_id=None, kernel_id=None, ramdisk_id=None, key_name=DEFAULT_KEYPAIR, placement=None, security_groups=['CloudMan']): """ Start an instance. If instance start was OK, return the ResultSet object diff --git a/templates/webapps/galaxy/cloud/index.mako b/templates/webapps/galaxy/cloud/index.mako index 9fc87d004d4..702027b1ef2 100644 --- a/templates/webapps/galaxy/cloud/index.mako +++ b/templates/webapps/galaxy/cloud/index.mako @@ -202,12 +202,14 @@ Credentials section of the AWS Console.
+ %if not share_string: + %endif

@@ -244,6 +246,10 @@ Credentials section of the AWS Console.
%endif + %if ami: + + %endif +
%endif + %if bucket_default: + + %endif +

+
- This is the text string that uniquely identifies your account, found in the + This is the text string that uniquely identifies your account, found in the Security Credentials section of the AWS Console.
-
+
This is your AWS Secret Key, also found in the Security Credentials section of the AWS Console.
@@ -205,7 +205,7 @@ Credentials section of the AWS Console.
%if not share_string: @@ -231,7 +231,7 @@ Credentials section of the AWS Console.
-
From dab318bacce9825bb73a3625affef1e7001c293a Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Tue, 4 Jun 2013 15:50:25 -0400 Subject: [PATCH 063/318] Add a not tested container for the automated tool test results in the tool shed. --- .../webapps/tool_shed/util/container_util.py | 18 ++++++++++ .../webapps/tool_shed/repository/common.mako | 34 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/lib/galaxy/webapps/tool_shed/util/container_util.py b/lib/galaxy/webapps/tool_shed/util/container_util.py index a20ed710793..5c610c3f65e 100644 --- a/lib/galaxy/webapps/tool_shed/util/container_util.py +++ b/lib/galaxy/webapps/tool_shed/util/container_util.py @@ -36,6 +36,7 @@ class Folder( object ): self.tool_dependencies = [] self.failed_tests = [] self.missing_test_components = [] + self.not_tested = [] self.passed_tests = [] self.test_environments = [] self.repository_dependencies = [] @@ -161,6 +162,14 @@ class MissingTestComponent( object ): self.tool_version = tool_version +class NotTested( object ): + """NotTested object""" + + def __init__( self, id=None, reason=None ): + self.id = id + self.reason = reason + + class PassedTest( object ): """Passed tool tests object""" @@ -1048,6 +1057,15 @@ def build_tool_test_results_folder( trans, folder_id, tool_test_results_dict, la tool_shed_mercurial_version=test_environment_dict.get( 'tool_shed_mercurial_version', '' ), tool_shed_revision=test_environment_dict.get( 'tool_shed_revision', '' ) ) folder.test_environments.append( test_environment ) + not_tested_dict = tool_test_results_dict.get( 'not_tested', {} ) + if not_tested_dict: + folder_id += 1 + folder = Folder( id=folder_id, key='not_tested', label='Not tested', parent=test_results_folder ) + test_results_folder.folders.append( folder ) + not_tested_id = 0 + not_tested = NotTested( id=not_tested_id, + reason=not_tested_dict.get( 'reason', '' ) ) + folder.not_tested.append( not_tested ) passed_tests_dicts = tool_test_results_dict.get( 'passed_tests', [] ) if passed_tests_dicts: folder_id += 1 diff --git a/templates/webapps/tool_shed/repository/common.mako b/templates/webapps/tool_shed/repository/common.mako index 1c7126bfdb8..085c3a267d5 100644 --- a/templates/webapps/tool_shed/repository/common.mako +++ b/templates/webapps/tool_shed/repository/common.mako @@ -337,6 +337,11 @@ ${render_failed_test( failed_test, pad, my_row, row_counter )} %endfor %endif + %if folder.not_tested: + %for not_tested in folder.not_tested: + ${render_not_tested( not_tested, pad, my_row, row_counter )} + %endfor + %endif %if folder.passed_tests: %for passed_test in folder.passed_tests: ${render_passed_test( passed_test, pad, my_row, row_counter )} @@ -731,6 +736,35 @@ %> +<%def name="render_not_tested( not_tested, pad, parent, row_counter, row_is_header=False )"> + <% encoded_id = trans.security.encode_id( not_tested.id ) %> + + + + + <% + my_row = row_counter.count + row_counter.increment() + %> + + <%def name="render_passed_test( passed_test, pad, parent, row_counter, row_is_header=False )"> <% encoded_id = trans.security.encode_id( passed_test.id ) %> +
+ + +
${not_tested.reason | h}
+
@@ -528,21 +528,21 @@ <%def name="render_missing_test_component( missing_test_component, pad, parent, row_counter, row_is_header=False )"> <% encoded_id = trans.security.encode_id( missing_test_component.id ) %> - +
Tool id: ${failed_test.tool_id | h}
@@ -659,21 +659,21 @@ <%def name="render_tool_dependency_installation_error( installation_error, pad, parent, row_counter, row_is_header=False )"> <% encoded_id = trans.security.encode_id( installation_error.id ) %> - +
Tool id: ${missing_test_component.tool_id | h}
@@ -697,21 +697,21 @@ <%def name="render_repository_installation_error( installation_error, pad, parent, row_counter, row_is_header=False, is_current_repository=False )"> <% encoded_id = trans.security.encode_id( installation_error.id ) %> - +
%if not is_current_repository: @@ -738,21 +738,21 @@ <%def name="render_not_tested( not_tested, pad, parent, row_counter, row_is_header=False )"> <% encoded_id = trans.security.encode_id( not_tested.id ) %> - +
@@ -767,21 +767,21 @@ <%def name="render_passed_test( passed_test, pad, parent, row_counter, row_is_header=False )"> <% encoded_id = trans.security.encode_id( passed_test.id ) %> - +
${not_tested.reason | h}
From 586c673338c7ec8dcf96df464d51400429983257 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Wed, 26 Jun 2013 21:17:00 -0400 Subject: [PATCH 227/318] Add support for repairing an installed tool shed repository and all of it's repository dependency hierarchy. Repairing a repository hierarchy will attempt to ensure all repositories in the hierarchy are correctly installed and all tool dependencies of each repository in the hierarchy are correctly installed. There is a new Repository Actions pop-up menu item for each repository labeled "Repair repository" that enables this in the browser. This is also supported in the enhanced Galaxy API using a command like the following from the ~/scripts/api directory: python ./repair_tool_shed_repository.py --api -l http://localhost:8763 --url http://localhost:9009/ -o test -r 1018e3cee313 --name chemicaltoolbox --- .../galaxy/api/tool_shed_repositories.py | 56 ++++- lib/galaxy/webapps/galaxy/buildapp.py | 13 +- .../galaxy/controllers/admin_toolshed.py | 108 +++++---- .../galaxy_install/repository_util.py | 226 ++++++++++++++++-- .../util/repository_dependency_util.py | 18 ++ lib/tool_shed/util/shed_util_common.py | 32 +-- lib/tool_shed/util/tool_dependency_util.py | 9 + scripts/api/install_tool_shed_repositories.py | 4 +- scripts/api/repair_tool_shed_repository.py | 61 +++++ .../repository_actions_menu.mako | 1 + 10 files changed, 437 insertions(+), 91 deletions(-) create mode 100644 scripts/api/repair_tool_shed_repository.py diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 434b58df2b7..3f2c27c7af5 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -9,6 +9,7 @@ from galaxy.web.base.controller import BaseAPIController from tool_shed.galaxy_install import repository_util from tool_shed.util import common_util +from tool_shed.util import encoding_util import tool_shed.util.shed_util_common as suc log = logging.getLogger( __name__ ) @@ -244,7 +245,7 @@ class ToolShedRepositoriesController( BaseAPIController ): # order the list of tsr_ids to ensure all repositories install in the required order. tsr_ids = [ trans.security.encode_id( tool_shed_repository.id ) for tool_shed_repository in tool_shed_repositories ] ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ - repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys ) + repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) # Install the repositories, keeping track of each one for later display. for index, tsr_id in enumerate( ordered_tsr_ids ): tool_shed_repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) @@ -358,3 +359,56 @@ class ToolShedRepositoriesController( BaseAPIController ): elif isinstance( installed_tool_shed_repositories, list ): all_installed_tool_shed_repositories.extend( installed_tool_shed_repositories ) return all_installed_tool_shed_repositories + + @web.expose_api + def repair_repository_revision( self, trans, payload, **kwd ): + """ + POST /api/tool_shed_repositories/repair_repository_revision + Repair a specified repository revision previously installed into Galaxy. + + :param key: the current Galaxy admin user's API key + + The following parameters are included in the payload. + :param tool_shed_url (required): the base URL of the Tool Shed from which the Repository was installed + :param name (required): the name of the Repository + :param owner (required): the owner of the Repository + :param changset_revision (required): the changset_revision of the RepositoryMetadata object associated with the Repository + """ + api_key = kwd.get( 'key', None ) + # Get the information about the repository to be installed from the payload. + tool_shed_url = payload.get( 'tool_shed_url', '' ) + if not tool_shed_url: + raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." ) + name = payload.get( 'name', '' ) + if not name: + raise HTTPBadRequest( detail="Missing required parameter 'name'." ) + owner = payload.get( 'owner', '' ) + if not owner: + raise HTTPBadRequest( detail="Missing required parameter 'owner'." ) + changeset_revision = payload.get( 'changeset_revision', '' ) + if not changeset_revision: + raise HTTPBadRequest( detail="Missing required parameter 'changeset_revision'." ) + tool_shed_repositories = [] + tool_shed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed_url, name, owner, changeset_revision ) + repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository ) + ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) + ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) + if ordered_tsr_ids and ordered_repo_info_dicts: + repositories_for_repair = [] + for index, tsr_id in enumerate( ordered_tsr_ids ): + repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) + repo_info_dict = ordered_repo_info_dicts[ index ] + # TODO: handle errors in repair_dict. + repair_dict = repository_util.repair_tool_shed_repository( trans, + repository, + encoding_util.tool_shed_encode( repo_info_dict ) ) + repository_dict = repository.get_api_value( value_mapper=default_tool_shed_repository_value_mapper( trans, repository ) ) + repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( repository.id ) ) + if repair_dict: + errors = repair_dict.get( repository.name, [] ) + repository_dict[ 'errors_attempting_repair' ] = ' '.join( errors ) + tool_shed_repositories.append( repository_dict ) + # Display the list of repaired repositories. + return tool_shed_repositories diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index d7041829f0b..675b2c23fd3 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -163,12 +163,13 @@ def app_factory( global_conf, **kwargs ): webapp.mapper.connect("workflow_dict", '/api/workflows/download/{workflow_id}', controller='workflows', action='workflow_dict', conditions=dict(method=['GET'])) # Galaxy API for tool shed features. webapp.mapper.resource( 'tool_shed_repository', - 'tool_shed_repositories', - controller='tool_shed_repositories', - name_prefix='tool_shed_repository_', - path_prefix='/api', - new={ 'install_repository_revision' : 'POST' }, - parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) ) + 'tool_shed_repositories', + member={ 'repair_repository_revision' : 'POST' }, + controller='tool_shed_repositories', + name_prefix='tool_shed_repository_', + path_prefix='/api', + new={ 'install_repository_revision' : 'POST' }, + parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) ) # Connect logger from app if app.trace_logger: webapp.trace_logger = app.trace_logger diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 6b2108d09da..e9d00ea1304 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -317,27 +317,6 @@ class AdminToolshed( AdminGalaxy ): trans.response.headers['Expires'] = '0' return suc.get_repository_file_contents( file_path ) - @web.expose - @web.require_admin - def get_repository_dependencies( self, trans, repository_id, repository_name, repository_owner, changeset_revision ): - """ - Send a request to the appropriate tool shed to retrieve the dictionary of repository dependencies defined for the received repository - name, owner and changeset revision. The received repository_id is the encoded id of the installed tool shed repository in Galaxy. We - need it so that we can derive the tool shed from which it was installed. - """ - repository = suc.get_installed_tool_shed_repository( trans, repository_id ) - tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) - url = suc.url_join( tool_shed_url, - 'repository/get_repository_dependencies?name=%s&owner=%s&changeset_revision=%s' % \ - ( repository_name, repository_owner, changeset_revision ) ) - raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) - if len( raw_text ) > 2: - encoded_text = json.from_json_string( raw_text ) - text = encoding_util.tool_shed_decode( encoded_text ) - else: - text = '' - return text - @web.expose @web.require_admin def get_tool_dependencies( self, trans, repository_id, repository_name, repository_owner, changeset_revision ): @@ -548,12 +527,7 @@ class AdminToolshed( AdminGalaxy ): def manage_repositories( self, trans, **kwd ): message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - tsrid = kwd.get( 'tool_shed_repository_id', None ) - tsridslist = util.listify( kwd.get( 'tool_shed_repository_ids', None ) ) - if not tsridslist: - tsridslist = util.listify( kwd.get( 'id', None ) ) - if tsrid and tsrid not in tsridslist: - tsridslist.append( tsrid ) + tsridslist = repository_util.get_tool_shed_repository_ids( **kwd ) if 'operation' in kwd: operation = kwd[ 'operation' ].lower() if not tsridslist: @@ -596,7 +570,7 @@ class AdminToolshed( AdminGalaxy ): # Some repositories may have repository dependencies that are required to be installed before the dependent repository, so we'll # order the list of tsr_ids to ensure all repositories install in the required order. ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ - repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys ) + repository_util.order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) for tsr_id in ordered_tsr_ids: repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) if repository.status in [ trans.model.ToolShedRepository.installation_status.NEW, @@ -615,6 +589,14 @@ class AdminToolshed( AdminGalaxy ): else: kwd[ 'message' ] = 'All selected tool shed repositories are already installed.' kwd[ 'status' ] = 'error' + elif operation == "repair": + # In this case, tsridslist is ordered. + repositories_for_repair = [] + for tsr_id in tsridslist: + repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) + repositories_for_repair.append( repository ) + ordered_repo_info_dicts = kwd.get( 'ordered_repo_info_dicts', [] ) + self.repair_tool_shed_repositories( trans, repositories_for_repair, ordered_repo_info_dicts ) return self.repository_installation_grid( trans, **kwd ) @web.expose @@ -799,14 +781,9 @@ class AdminToolshed( AdminGalaxy ): @web.expose @web.require_admin def monitor_repository_installation( self, trans, **kwd ): - tsrid = kwd.get( 'tool_shed_repository_id', None ) - tsridslist = util.listify( kwd.get( 'tool_shed_repository_ids', None ) ) + tsridslist = repository_util.get_tool_shed_repository_ids( **kwd ) if not tsridslist: - tsridslist = util.listify( kwd.get( 'id', None ) ) - if tsrid and tsrid not in tsridslist: - tsridslist.append( tsrid ) - if not tsridslist: - tsridslist = get_ids_of_tool_shed_repositories_being_installed( trans, as_string=False ) + tsridslist = suc.get_ids_of_tool_shed_repositories_being_installed( trans, as_string=False ) kwd[ 'tool_shed_repository_ids' ] = tsridslist return self.repository_installation_grid( trans, **kwd ) @@ -1097,11 +1074,7 @@ class AdminToolshed( AdminGalaxy ): else: # Entering this else block occurs only if the tool_shed_repository does not include any valid tools. if install_repository_dependencies: - repository_dependencies = self.get_repository_dependencies( trans=trans, - repository_id=repository_id, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - changeset_revision=tool_shed_repository.changeset_revision ) + repository_dependencies = repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans, tool_shed_repository ) else: repository_dependencies = None if metadata: @@ -1174,10 +1147,28 @@ class AdminToolshed( AdminGalaxy ): Inspect the repository dependency hierarchy for a specified repository and attempt to make sure they are all properly installed as well as each repository's tool dependencies. """ + # TODO: figure out how to handle installing repositories and tool dependencies consecutively since each redirects to an ajaxian grid. This + # is not a problem in that Galaxy API. message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) - repository_id = kwd[ 'id' ] + repository_id = kwd.get( 'id', None ) + if not repository_id: + message = 'Invalid installed tool shed repository id %s received.' % str( repository_id ) + status = 'error' + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='browse_repositories', + message=message, + status=status ) ) tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository ) + ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) + ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) + if ordered_tsr_ids and ordered_repo_info_dicts: + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='manage_repositories', + operation='repair', + ordered_tsr_ids=ordered_tsr_ids, + ordered_repo_info_dicts=ordered_repo_info_dicts ) ) @web.json def repository_installation_status_updates( self, trans, ids=None, status_list=None ): @@ -1200,6 +1191,21 @@ class AdminToolshed( AdminGalaxy ): 'utf-8' ) ) ) return rval + @web.expose + @web.require_admin + def repair_tool_shed_repositories( self, trans, tool_shed_repositories, repo_info_dicts, **kwd ): + """Repair specified tool shed repositories.""" + # The received lists of tool_shed_repositories and repo_info_dicts are ordered. + for index, tool_shed_repository in enumerate( tool_shed_repositories ): + repo_info_dict = repo_info_dicts[ index ] + repair_dict = repository_util.repair_tool_shed_repository( trans, + tool_shed_repository, + encoding_util.tool_shed_encode( repo_info_dict ) ) + tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='monitor_repository_installation', + tool_shed_repository_ids=tsr_ids_for_monitoring ) ) + @web.expose @web.require_admin def reselect_tool_panel_section( self, trans, **kwd ): @@ -1268,11 +1274,7 @@ class AdminToolshed( AdminGalaxy ): raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) readme_files_dict = json.from_json_string( raw_text ) tool_dependencies = metadata.get( 'tool_dependencies', None ) - repository_dependencies = self.get_repository_dependencies( trans=trans, - repository_id=repository_id, - repository_name=tool_shed_repository.name, - repository_owner=tool_shed_repository.owner, - changeset_revision=tool_shed_repository.changeset_revision ) + repository_dependencies = repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans, tool_shed_repository ) repo_info_dict = repository_util.create_repo_info_dict( trans=trans, repository_clone_url=repository_clone_url, changeset_revision=tool_shed_repository.changeset_revision, @@ -1439,13 +1441,13 @@ class AdminToolshed( AdminGalaxy ): """An error occurred while cloning the repository, so reset everything necessary to enable another attempt.""" repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) if kwd.get( 'reset_repository', False ): - suc.set_repository_attributes( trans, - repository, - status=trans.model.ToolShedRepository.installation_status.NEW, - error_message=None, - deleted=False, - uninstalled=False, - remove_from_disk=True ) + repository_util.set_repository_attributes( trans, + repository, + status=trans.model.ToolShedRepository.installation_status.NEW, + error_message=None, + deleted=False, + uninstalled=False, + remove_from_disk=True ) new_kwd = {} new_kwd[ 'message' ] = "You can now attempt to install the repository named %s again." % repository.name new_kwd[ 'status' ] = "done" diff --git a/lib/tool_shed/galaxy_install/repository_util.py b/lib/tool_shed/galaxy_install/repository_util.py index 53c4406231d..302c10ed89b 100644 --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -6,6 +6,7 @@ import threading from galaxy import tools from galaxy.util import asbool from galaxy.util import json +from galaxy import util from galaxy import web from galaxy.model.orm import or_ from galaxy.webapps.tool_shed.util import container_util @@ -93,6 +94,30 @@ def create_repo_info_dict( trans, repository_clone_url, changeset_revision, ctx_ tool_dependencies ) return repo_info_dict +def get_installed_repositories_from_repository_dependencies( trans, repository_dependencies_dict ): + installed_repositories = [] + for rd_key, rd_vals in repository_dependencies_dict.items(): + if rd_key in [ 'root_key', 'description' ]: + continue + # rd_key is something like: 'http://localhost:9009__ESEP__package_rdkit_2012_12__ESEP__test__ESEP__d635ffb9c665__ESEP__True' + # rd_val is something like: [['http://localhost:9009', 'package_numpy_1_7', 'test', 'cddd64ecd985', 'True']] + try: + tool_shed, name, owner, changeset_revision, prior_installation_required = container_util.get_components_from_key( rd_key ) + except: + tool_shed, name, owner, changeset_revision = container_util.get_components_from_key( rd_val ) + installed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed, name, owner, changeset_revision ) + if installed_repository not in installed_repositories: + installed_repositories.append( installed_repository ) + for rd_val in rd_vals: + try: + tool_shed, name, owner, changeset_revision, prior_installation_required = container_util.get_components_from_key( rd_key ) + except: + tool_shed, name, owner, changeset_revision = container_util.get_components_from_key( rd_val ) + installed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed, name, owner, changeset_revision ) + if installed_repository not in installed_repositories: + installed_repositories.append( installed_repository ) + return installed_repositories + def get_next_prior_install_required_dict_entry( prior_install_required_dict, processed_tsr_ids ): """ The order in which the prior_install_required_dict is processed is critical in order to ensure that the ultimate repository installation order is correctly @@ -142,6 +167,51 @@ def get_prior_install_required_dict( trans, tsr_ids, repo_info_dicts ): prior_install_required_dict[ encoded_repository_id ] = prior_install_ids return prior_install_required_dict +def get_repair_dict( trans, repository ): + """ + Inspect the installed repository dependency hierarchy for a specified repository and attempt to make sure they are all properly installed as well as + each repository's tool dependencies. This method is called only from Galaxy when attempting to correct issues with an installed repository that has + installation problems somewhere in it's dependency hierarchy. + """ + repair_dict = {} + # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. + repository_dependencies_dict = repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans, repository ) + # Generate the list of installed repositories from the information contained in the repository_dependencies dictionary. + installed_repositories = get_installed_repositories_from_repository_dependencies( trans, repository_dependencies_dict ) + # Some repositories may have repository dependencies that are required to be installed before the dependent repository, so we'll order the list of + # tsr_ids to ensure all repositories are repaired in the required order. + tsr_ids = [] + repo_info_dicts = [] + for installed_repository in installed_repositories: + tsr_ids.append( trans.security.encode_id( installed_repository.id ) ) + repository_clone_url = suc.generate_clone_url_for_installed_repository( trans.app, installed_repository ) + repository_dependencies = repository_dependency_util.get_repository_dependencies_for_installed_tool_shed_repository( trans, installed_repository ) + metadata = installed_repository.metadata + if metadata: + tool_dependencies = metadata.get( 'tool_dependencies', None ) + else: + tool_dependencies = None + repo_info_dict = create_repo_info_dict( trans=trans, + repository_clone_url=repository_clone_url, + changeset_revision=installed_repository.changeset_revision, + ctx_rev=installed_repository.ctx_rev, + repository_owner=installed_repository.owner, + repository_name=installed_repository.name, + repository=None, + repository_metadata=None, + tool_dependencies=tool_dependencies, + repository_dependencies=repository_dependencies ) + repo_info_dicts.append( repo_info_dict ) + # We don't consider tool_panel_section_keys since the repository database records already exist. + ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = order_components_for_installation( trans, + tsr_ids, + repo_info_dicts, + tool_panel_section_keys=None ) + repair_dict[ 'ordered_tsr_ids' ] = ordered_tsr_ids + repair_dict[ 'ordered_repo_info_dicts' ] = ordered_repo_info_dicts + repair_dict[ 'ordered_tool_panel_section_keys' ] = ordered_tool_panel_section_keys + return repair_dict + def get_repo_info_dict( trans, repository_id, changeset_revision ): repository = suc.get_repository_in_tool_shed( trans, repository_id ) repository_clone_url = suc.generate_clone_url_for_repository_in_tool_shed( trans, repository ) @@ -175,7 +245,7 @@ def get_repo_info_dict( trans, repository_id, changeset_revision ): repository_dependencies=None ) return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, has_repository_dependencies -def get_repository_components_for_installation( encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys ): +def get_repository_components_for_installation( encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys=None ): """ The received encoded_tsr_ids, repo_info_dicts, and tool_panel_section_keys are 3 lists that contain associated elements at each location in the list. This method will return the elements from repo_info_dicts and tool_panel_section_keys associated with the received encoded_tsr_id @@ -184,7 +254,10 @@ def get_repository_components_for_installation( encoded_tsr_id, encoded_tsr_ids, for index, tsr_id in enumerate( encoded_tsr_ids ): if tsr_id == encoded_tsr_id: repo_info_dict = repo_info_dicts[ index ] - tool_panel_section_key = tool_panel_section_keys[ index ] + if tool_panel_section_keys: + tool_panel_section_key = tool_panel_section_keys[ index ] + else: + tool_panel_section_key = None return repo_info_dict, tool_panel_section_key return None, None @@ -220,6 +293,26 @@ def get_repository_ids_requiring_prior_install( trans, tsr_ids, repository_depen prior_install_ids.append( encoded_repository_id ) return prior_install_ids +def get_tool_shed_repository_ids( as_string=False, **kwd ): + tsrid = kwd.get( 'tool_shed_repository_id', None ) + tsridslist = util.listify( kwd.get( 'tool_shed_repository_ids', None ) ) + if not tsridslist: + tsridslist = util.listify( kwd.get( 'id', None ) ) + if tsridslist: + if tsrid and tsrid not in tsridslist: + tsridslist.append( tsrid ) + if as_string: + return ','.join( tsridslist ) + return tsridslist + else: + tsridslist = util.listify( kwd.get( 'ordered_tsr_ids', None ) ) + if as_string: + return ','.join( tsridslist ) + return tsridslist + if as_string: + '' + return [] + def get_update_to_changeset_revision_and_ctx_rev( trans, repository ): """Return the changeset revision hash to which the repository can be updated.""" changeset_revision_dict = {} @@ -447,7 +540,7 @@ def install_tool_shed_repository( trans, tool_shed_repository, repo_info_dict, t cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) if cloned_ok: if reinstalling: - # Since we're reinstalling the repository we need to find the latest changeset revision to which is can be updated. + # Since we're reinstalling the repository we need to find the latest changeset revision to which it can be updated. changeset_revision_dict = get_update_to_changeset_revision_and_ctx_rev( trans, tool_shed_repository ) current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None ) current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None ) @@ -503,13 +596,13 @@ def install_tool_shed_repository( trans, tool_shed_repository, repo_info_dict, t suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED ) else: # An error occurred while cloning the repository, so reset everything necessary to enable another attempt. - suc.set_repository_attributes( trans, - tool_shed_repository, - status=trans.model.ToolShedRepository.installation_status.ERROR, - error_message=error_message, - deleted=False, - uninstalled=False, - remove_from_disk=True ) + set_repository_attributes( trans, + tool_shed_repository, + status=trans.model.ToolShedRepository.installation_status.ERROR, + error_message=error_message, + deleted=False, + uninstalled=False, + remove_from_disk=True ) def merge_containers_dicts_for_new_install( containers_dicts ): """ @@ -591,7 +684,7 @@ def merge_containers_dicts_for_new_install( containers_dicts ): lock.release() return new_containers_dict -def order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys ): +def order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys=None ): """ Some repositories may have repository dependencies that are required to be installed before the dependent repository. This method will inspect the list of repositories about to be installed and make sure to order them appropriately. For each repository about to be installed, if required repositories are not @@ -617,14 +710,19 @@ def order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_pan prior_repo_info_dict, prior_tool_panel_section_key = get_repository_components_for_installation( prior_install_required_id, tsr_ids, repo_info_dicts, - tool_panel_section_keys ) + tool_panel_section_keys=tool_panel_section_keys ) ordered_tsr_ids.append( prior_install_required_id ) ordered_repo_info_dicts.append( prior_repo_info_dict ) - ordered_tool_panel_section_keys.append( prior_tool_panel_section_key ) - repo_info_dict, tool_panel_section_key = get_repository_components_for_installation( tsr_id, tsr_ids, repo_info_dicts, tool_panel_section_keys ) + if tool_panel_section_keys: + ordered_tool_panel_section_keys.append( prior_tool_panel_section_key ) + repo_info_dict, tool_panel_section_key = get_repository_components_for_installation( tsr_id, + tsr_ids, + repo_info_dicts, + tool_panel_section_keys=tool_panel_section_keys ) ordered_tsr_ids.append( tsr_id ) ordered_repo_info_dicts.append( repo_info_dict ) - ordered_tool_panel_section_keys.append( tool_panel_section_key ) + if tool_panel_section_keys: + ordered_tool_panel_section_keys.append( tool_panel_section_key ) return ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys def populate_containers_dict_for_new_install( trans, tool_shed_url, tool_path, readme_files_dict, installed_repository_dependencies, missing_repository_dependencies, @@ -663,3 +761,101 @@ def populate_containers_dict_for_new_install( trans, tool_shed_url, tool_path, r def pull_repository( repo, repository_clone_url, ctx_rev ): """Pull changes from a remote repository to a local one.""" commands.pull( suc.get_configured_ui(), repo, source=repository_clone_url, rev=[ ctx_rev ] ) + +def repair_tool_shed_repository( trans, repository, repo_info_dict ): + + def add_repair_dict_entry( repository_name, error_message ): + if repository_name in repair_dict: + repair_dict[ repository_name ].append( error_message ) + else: + repair_dict[ repository_name ] = [ error_message ] + return repair_dict + + metadata = repository.metadata + repair_dict = {} + if repository.status in [ trans.model.ToolShedRepository.installation_status.DEACTIVATED ]: + try: + common_install_util.activate_repository( trans, repository ) + except Exception, e: + error_message = "Error activating repository %s: %s" % ( repository.name, str( e ) ) + log.debug( error_message ) + repair_dict [ repository.name ] = error_message + elif repository.status not in [ trans.model.ToolShedRepository.installation_status.INSTALLED ]: + # TODO: this may cause problems if the repository is currently being installed. + shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) + # Reset the repository attributes to the New state for installation. + if metadata: + tool_panel_section_key = tool_util.handle_tool_panel_selection( trans, + metadata, + no_changes_checked=True, + tool_panel_section=None, + new_tool_panel_section=None ) + else: + # The tools will be loaded outside of any sections in the tool panel. + tool_panel_section_key = None + set_repository_attributes( trans, + repository, + status=trans.model.ToolShedRepository.installation_status.NEW, + error_message=None, + deleted=False, + uninstalled=False, + remove_from_disk=True ) + install_tool_shed_repository( trans, + repository, + repo_info_dict, + tool_panel_section_key, + shed_tool_conf, + tool_path, + install_tool_dependencies=True, + reinstalling=True ) + if repository.status in [ trans.model.ToolShedRepository.installation_status.ERROR ]: + repair_dict = add_repair_dict_entry( repository.name, repository.error_message ) + else: + # We have an installed tool shed repository, so handle tool dependencies if necessary. + if repository.missing_tool_dependencies and metadata and 'tool_dependencies' in metadata: + work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itdep" ) + # Reset missing tool dependencies. + for tool_dependency in repository.missing_tool_dependencies: + if tool_dependency.status in [ trans.model.ToolDependency.installation_status.ERROR ]: + tool_dependency_util.set_tool_dependency_attributes( trans, + tool_dependency, + trans.model.ToolDependency.installation_status.UNINSTALLED, + None, + remove_from_disk=True ) + trans.sa_session.refresh( tool_dependency ) + # Install tool dependencies. + suc.update_tool_shed_repository_status( trans.app, + repository, + trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) + # Get the tool_dependencies.xml file from the repository. + tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repository.repo_path( trans.app ) ) + installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app, + tool_shed_repository=repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=repository.tool_dependencies ) + for installed_tool_dependency in installed_tool_dependencies: + if installed_tool_dependency.status in [ trans.model.ToolDependency.installation_status.ERROR ]: + repair_dict = add_repair_dict_entry( repository.name, installed_tool_dependency.error_message ) + try: + shutil.rmtree( work_dir ) + except: + pass + suc.update_tool_shed_repository_status( trans.app, repository, trans.model.ToolShedRepository.installation_status.INSTALLED ) + return repair_dict + +def set_repository_attributes( trans, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): + if remove_from_disk: + relative_install_dir = repository.repo_path( trans.app ) + if relative_install_dir: + clone_dir = os.path.abspath( relative_install_dir ) + try: + shutil.rmtree( clone_dir ) + log.debug( "Removed repository installation directory: %s" % str( clone_dir ) ) + except Exception, e: + log.debug( "Error removing repository installation directory %s: %s" % ( str( clone_dir ), str( e ) ) ) + repository.error_message = error_message + repository.status = status + repository.deleted = deleted + repository.uninstalled = uninstalled + trans.sa_session.add( repository ) + trans.sa_session.flush() diff --git a/lib/tool_shed/util/repository_dependency_util.py b/lib/tool_shed/util/repository_dependency_util.py index ab425404cd8..e2f561930cf 100644 --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -4,6 +4,7 @@ from galaxy import eggs from galaxy.util import json from galaxy.webapps.tool_shed.util import container_util import tool_shed.util.shed_util_common as suc +from tool_shed.util import common_util from tool_shed.util import common_install_util from tool_shed.util import encoding_util from tool_shed.util import metadata_util @@ -285,6 +286,23 @@ def get_prior_installation_required_for_key( toolshed_base_url, repository, repo # Default prior_installation_required to False. return False +def get_repository_dependencies_for_installed_tool_shed_repository( trans, repository ): + """ + Send a request to the appropriate tool shed to retrieve the dictionary of repository dependencies defined for the received repository which is + installed into Galaxy. This method is called only from Galaxy. + """ + tool_shed_url = suc.get_url_from_tool_shed( trans.app, repository.tool_shed ) + url = suc.url_join( tool_shed_url, + 'repository/get_repository_dependencies?name=%s&owner=%s&changeset_revision=%s' % \ + ( str( repository.name ), str( repository.owner ), str( repository.changeset_revision ) ) ) + raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url ) + if len( raw_text ) > 2: + encoded_text = json.from_json_string( raw_text ) + text = encoding_util.tool_shed_decode( encoded_text ) + else: + text = '' + return text + def get_repository_dependencies_for_changeset_revision( trans, repository, repository_metadata, toolshed_base_url, key_rd_dicts_to_be_processed=None, all_repository_dependencies=None, handled_key_rd_dicts=None, circular_repository_dependencies=None ): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 0475795ddd5..b075391501a 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -547,6 +547,24 @@ def get_file_context_from_ctx( ctx, filename ): return 'DELETED' return None +def get_ids_of_tool_shed_repositories_being_installed( trans, as_string=False ): + installing_repository_ids = [] + new_status = trans.model.ToolShedRepository.installation_status.NEW + cloning_status = trans.model.ToolShedRepository.installation_status.CLONING + setting_tool_versions_status = trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS + installing_dependencies_status = trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES + loading_datatypes_status = trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES + for tool_shed_repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( or_( trans.model.ToolShedRepository.status == new_status, + trans.model.ToolShedRepository.status == cloning_status, + trans.model.ToolShedRepository.status == setting_tool_versions_status, + trans.model.ToolShedRepository.status == installing_dependencies_status, + trans.model.ToolShedRepository.status == loading_datatypes_status ) ): + installing_repository_ids.append( trans.security.encode_id( tool_shed_repository.id ) ) + if as_string: + return ','.join( installing_repository_ids ) + return installing_repository_ids + def get_installed_tool_shed_repository( trans, id ): """Get a tool shed repository record from the Galaxy database defined by the id.""" return trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) ) @@ -1223,20 +1241,6 @@ def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revi """Return a reversed list of changesets in the repository changelog up to and including the included_upper_bounds_changeset_revision.""" return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) -def set_repository_attributes( trans, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): - if remove_from_disk: - relative_install_dir = repository.repo_path( trans.app ) - if relative_install_dir: - clone_dir = os.path.abspath( relative_install_dir ) - shutil.rmtree( clone_dir ) - log.debug( "Removed repository installation directory: %s" % str( clone_dir ) ) - repository.error_message = error_message - repository.status = status - repository.deleted = deleted - repository.uninstalled = uninstalled - trans.sa_session.add( repository ) - trans.sa_session.flush() - def set_prior_installation_required( repository, required_repository ): """Return True if the received required_repository must be installed before the received repository.""" # This method is called only from Galaxy when rendering repository dependencies for an installed tool shed repository. diff --git a/lib/tool_shed/util/tool_dependency_util.py b/lib/tool_shed/util/tool_dependency_util.py index ea2f2b47e8c..1b3b2112442 100644 --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -344,3 +344,12 @@ def remove_tool_dependency_installation_directory( dependency_install_dir ): removed = True error_message = '' return removed, error_message + +def set_tool_dependency_attributes( trans, tool_dependency, status, error_message, remove_from_disk=False ): + if remove_from_disk: + installation_directory = tool_dependency.installation_directory( trans.app ) + removed, err_msg = remove_tool_dependency_installation_directory( installation_directory ) + tool_dependency.error_message = error_message + tool_dependency.status = status + trans.sa_session.add( tool_dependency ) + trans.sa_session.flush() diff --git a/scripts/api/install_tool_shed_repositories.py b/scripts/api/install_tool_shed_repositories.py index 508b8236ce1..0bb86e43d40 100644 --- a/scripts/api/install_tool_shed_repositories.py +++ b/scripts/api/install_tool_shed_repositories.py @@ -10,7 +10,7 @@ This example requires a tool panel config file (e.g., tool_conf.xml, shed_tool_c Here is a working example of how to use this script to install a repository from the test tool shed. -./install_repository_tools.py --api --local --url http://testtoolshed.g2.bx.psu.edu --name gregs_filter --owner greg --revision f28d5018f9cb --tool-deps +./install_tool_shed_repositories.py --api --local --url http://testtoolshed.g2.bx.psu.edu --name gregs_filter --owner greg --revision f28d5018f9cb --tool-deps """ import os @@ -34,7 +34,7 @@ def main( options ): data[ 'install_repository_dependencies' ] = options.install_repository_dependencies if options.install_tool_dependencies: data[ 'install_tool_dependencies' ] = options.install_tool_dependencies - submit( options.api, '%s%s' % ( options.local_url.strip('/'), '/api/tool_shed_repositories/new/install_repository_revision' ), data ) + submit( options.api, '%s%s' % ( options.local_url.strip( '/' ), '/api/tool_shed_repositories/new/install_repository_revision' ), data ) if __name__ == '__main__': parser = argparse.ArgumentParser( description='Installation of tool shed repositories via the Galaxy API.' ) diff --git a/scripts/api/repair_tool_shed_repository.py b/scripts/api/repair_tool_shed_repository.py new file mode 100644 index 00000000000..ddb6de389a8 --- /dev/null +++ b/scripts/api/repair_tool_shed_repository.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +""" +Repair a specified repository revision previously installed into Galaxy. + + +Here is a working example of how to use this script to repair a repository installed into Galaxy. +./repair_tool_shed_repository.py --api --local --url http://testtoolshed.g2.bx.psu.edu --name gregs_filter --owner greg --revision f28d5018f9cb +""" + +import os +import sys +import argparse +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import display +from common import submit + +def clean_url( url ): + if url.find( '//' ) > 0: + # We have an url that includes a protocol, something like: http://localhost:9009 + items = url.split( '//' ) + return items[ 1 ].rstrip( '/' ) + return url.rstrip( '/' ) + +def main( options ): + """Collect all user data and install the tools via the Galaxy API.""" + api_key = options.api + base_galaxy_url = options.local_url.rstrip( '/' ) + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + cleaned_tool_shed_url = clean_url( base_tool_shed_url ) + installed_tool_shed_repositories_url = '%s/api/%s' % ( base_galaxy_url, 'tool_shed_repositories' ) + data = {} + data[ 'tool_shed_url' ] = cleaned_tool_shed_url + data[ 'name' ] = options.name + data[ 'owner' ] = options.owner + data[ 'changeset_revision' ] = options.changeset_revision + tool_shed_repository_id = None + installed_tool_shed_repositories = display( api_key, installed_tool_shed_repositories_url, return_formatted=False ) + for installed_tool_shed_repository in installed_tool_shed_repositories: + tool_shed = str( installed_tool_shed_repository[ 'tool_shed' ] ) + name = str( installed_tool_shed_repository[ 'name' ] ) + owner = str( installed_tool_shed_repository[ 'owner' ] ) + changeset_revision = str( installed_tool_shed_repository[ 'changeset_revision' ] ) + if tool_shed == cleaned_tool_shed_url and name == options.name and owner == options.owner and changeset_revision == options.changeset_revision: + tool_shed_repository_id = installed_tool_shed_repository[ 'id' ] + break + if tool_shed_repository_id: + url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/repair_repository_revision' % str( tool_shed_repository_id ) ) + submit( options.api, url, data ) + else: + print "Invalid tool_shed / name / owner / changeset_revision." + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Installation of tool shed repositories via the Galaxy API.' ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-l", "--local", dest="local_url", required=True, help="URL of the galaxy instance." ) + parser.add_argument( "-n", "--name", required=True, help="Repository name." ) + parser.add_argument( "-o", "--owner", required=True, help="Repository owner." ) + parser.add_argument( "-r", "--revision", dest="changeset_revision", required=True, help="Repository owner." ) + options = parser.parse_args() + main( options ) diff --git a/templates/admin/tool_shed_repository/repository_actions_menu.mako b/templates/admin/tool_shed_repository/repository_actions_menu.mako index e88a422cb88..42132fb9ca0 100644 --- a/templates/admin/tool_shed_repository/repository_actions_menu.mako +++ b/templates/admin/tool_shed_repository/repository_actions_menu.mako @@ -24,6 +24,7 @@ Manage repository Browse repository files Get repository updates + Repair repository %if repository.can_reset_metadata: Reset repository metadata %endif From 07658d8c33804552516bd3a6c0d423c889c6e91d Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Thu, 27 Jun 2013 10:17:04 -0400 Subject: [PATCH 228/318] Fix path issue that was introduced with the refactoring of common_util.url_download. --- lib/tool_shed/galaxy_install/tool_dependencies/common_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py index 526dfdf7df7..45b11ed3949 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -248,9 +248,9 @@ def url_download( install_dir, downloaded_file_name, download_url, extract=True zip_archive_extracted = extract_zip( file_path, install_dir ) dir = zip_extraction_directory( install_dir, downloaded_file_name ) else: - dir = install_dir + dir = os.path.abspath( install_dir ) else: - dir = install_dir + dir = os.path.abspath( install_dir ) return dir def zip_extraction_directory( file_path, file_name ): From 6d3b963ad1582aab669bf9ed15c7e0d57546d4c2 Mon Sep 17 00:00:00 2001 From: Aysam Guerler Date: Thu, 27 Jun 2013 11:54:19 -0400 Subject: [PATCH 229/318] Fix IE compatibility --- static/scripts/utils/galaxy.css.js | 2 +- static/style/blue/galaxy.frame.css | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/static/scripts/utils/galaxy.css.js b/static/scripts/utils/galaxy.css.js index 4e9ebfc56fe..c799533efdf 100644 --- a/static/scripts/utils/galaxy.css.js +++ b/static/scripts/utils/galaxy.css.js @@ -15,7 +15,7 @@ function get_attribute (classname, name) el.appendTo(':eq(0)'); // get value - var value = el.css(name).replace(/[^-\d\.]/g, ''); + var value = el.css(name); // remove element el.remove(); diff --git a/static/style/blue/galaxy.frame.css b/static/style/blue/galaxy.frame.css index 1efab24aaf9..f2b0cb829e4 100644 --- a/static/style/blue/galaxy.frame.css +++ b/static/style/blue/galaxy.frame.css @@ -48,11 +48,6 @@ z-index : 14002; overflow : hidden; position : absolute; - top : 0px; - left : 0px; - height : 100%; - width : 100%; - opacity : 1.0; background : #FFFFFF; border : 1px solid #D0D0D0; -webkit-box-shadow: 0 0 5px rgba(0,0,0,0.3); @@ -88,7 +83,6 @@ position : absolute; top : 8px; right : 120px; - padding : 0px 0px 0px 0px; cursor : pointer; color : #D0D0D0; } @@ -102,7 +96,6 @@ position : absolute; top : 6px; right : 165px; - padding : 0px 0px 0px 0px; cursor : pointer; color : #BCC800; } From 56557f78a0631133a54283c188aa4606bbff0ce7 Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Thu, 27 Jun 2013 12:11:21 -0400 Subject: [PATCH 230/318] Fix unicode issues in page editor. --- templates/webapps/galaxy/page/editor.mako | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/webapps/galaxy/page/editor.mako b/templates/webapps/galaxy/page/editor.mako index 426a72a483a..bb6e56faa18 100644 --- a/templates/webapps/galaxy/page/editor.mako +++ b/templates/webapps/galaxy/page/editor.mako @@ -782,7 +782,7 @@
- +
From 2e88d47e0d70b95ddb86aef10de980baff4ce13d Mon Sep 17 00:00:00 2001 From: Aysam Guerler Date: Thu, 27 Jun 2013 14:35:37 -0400 Subject: [PATCH 231/318] Add VCF visualization support for dataset viewer --- static/scripts/mvc/data.js | 117 ++++++++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 34 deletions(-) diff --git a/static/scripts/mvc/data.js b/static/scripts/mvc/data.js index 0a3979f755f..ae68cdf646c 100644 --- a/static/scripts/mvc/data.js +++ b/static/scripts/mvc/data.js @@ -120,7 +120,7 @@ var TabularDatasetChunkedView = Backbone.View.extend({ initialize: function(options) { // load trackster button - (new TabularButtonTracksterView(options)).render(); + new TabularButtonTracksterView(options); }, render: function() @@ -252,23 +252,63 @@ var TabularButtonTracksterView = Backbone.View.extend( // database key genome_build: null, + // data type + data_type: null, + // backbone initialize initialize: function (options) { - // verify that metadata exists - var metadata = options.model.attributes.metadata.attributes; - if (typeof metadata.chromCol === "undefined" || typeof metadata.startCol === "undefined" || typeof metadata.endCol === "undefined") - console.log("TabularButtonTrackster : Metadata for column identification is missing."); - else { - // read in columns - this.col.chrom = metadata.chromCol - 1; - this.col.start = metadata.startCol - 1; - this.col.end = metadata.endCol - 1; + // get options + var attributes = options.model.attributes; + var metadata = options.model.attributes.metadata.attributes; + + // check for datatype + if (typeof attributes.data_type !== "undefined") + this.data_type = attributes.data_type; + else + console.log("TabularButtonTrackster : Data type missing."); + + // check for bed-file format + if (this.data_type == "bed") + { + // verify that metadata exists + if (typeof metadata.chromCol !== "undefined" || typeof metadata.startCol !== "undefined" || typeof metadata.endCol !== "undefined") + { + // read in columns + this.col.chrom = metadata.chromCol - 1; + this.col.start = metadata.startCol - 1; + this.col.end = metadata.endCol - 1; + } else + console.log("TabularButtonTrackster : Bed-file metadata incomplete."); + } + + // check for vcf-file format + if (this.data_type == "vcf") + { + // search array + function search (str, array) + { + for (var j = 0; j < array.length; j++) + if (array[j].match(str)) return j; + return -1; + }; + + // load + this.col.chrom = search("Chrom", metadata.column_names); + this.col.start = search("Pos", metadata.column_names); + this.col.end = null; + + // verify that metadata exists + if (this.col.chrom == -1 || this.col.start == -1) + console.log("TabularButtonTrackster : VCF-file metadata incomplete."); } // check if(this.col.chrom === null) + { + console.log("TabularButtonTrackster : Chromosome column undefined."); return; + } // get dataset id if (typeof options.model.attributes.id === "undefined") @@ -285,6 +325,19 @@ var TabularButtonTracksterView = Backbone.View.extend( // get genome_build / database key if (typeof options.model.attributes.genome_build !== "undefined") this.genome_build = options.model.attributes.genome_build; + + // render the icon from template + var btn_viz = new IconButtonView({ model : new IconButton({ + title : 'Visualize', + icon_class : 'chart_curve', + id : 'btn_viz' + })}); + + // add it to the screen + this.$el.append(btn_viz.render().$el); + + // hide the button + $('#btn_viz').hide(); }, // backbone events @@ -297,29 +350,39 @@ var TabularButtonTracksterView = Backbone.View.extend( // show button btn_viz_show: function (e) { + // is numeric + function is_numeric(n) + { + return !isNaN(parseFloat(n)) && isFinite(n); + }; + // check if(this.col.chrom === null) return; - + // get selected data line var row = $(e.target).parent(); - + // verify that location has been found var chrom = row.children().eq(this.col.chrom).html(); var start = row.children().eq(this.col.start).html(); - var end = row.children().eq(this.col.end).html(); - if (chrom !== "" && start !== "" && end !== "") + + // end is optional + var end = this.col.end ? row.children().eq(this.col.end).html() : start; + + // double check location + if (!chrom.match("^#") && chrom !== "" && is_numeric(start)) { // get target gene region var btn_viz_pars = { dataset_id : this.dataset_id, gene_region : chrom + ":" + start + "-" + end }; - + // get button position var offset = row.offset(); var left = offset.left - 10; - var top = offset.top; + var top = offset.top - $(window).scrollTop(); // update css $('#btn_viz').css({'position': 'fixed', 'top': top + 'px', 'left': left + 'px'}); @@ -328,6 +391,9 @@ var TabularButtonTracksterView = Backbone.View.extend( // show the button $('#btn_viz').show(); + } else { + // hide the button + $('#btn_viz').hide(); } }, @@ -401,24 +467,7 @@ var TabularButtonTracksterView = Backbone.View.extend( }); return false; }; - }, - - // render frame - render: function() - { - // render the icon from template - var btn_viz = new IconButtonView({ model : new IconButton({ - title : 'Visualize', - icon_class : 'chart_curve', - id : 'btn_viz' - })}); - - // add it to the screen - this.$el.append(btn_viz.render().$el); - - // hide the button - $('#btn_viz').hide(); - } + } }); // -- Utility functions. -- From 24e6856dec07b11e6e1a331d418ce9665e439fab Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Thu, 27 Jun 2013 15:52:45 -0400 Subject: [PATCH 232/318] When repairing a repository via the browser, display the ordered list of repositories that will be inspected and repaired. --- .../galaxy/controllers/admin_toolshed.py | 26 +++++++--- .../repair_repository.mako | 51 +++++++++++++++++++ 2 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 templates/admin/tool_shed_repository/repair_repository.mako diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index e9d00ea1304..b3212b55d03 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1147,8 +1147,6 @@ class AdminToolshed( AdminGalaxy ): Inspect the repository dependency hierarchy for a specified repository and attempt to make sure they are all properly installed as well as each repository's tool dependencies. """ - # TODO: figure out how to handle installing repositories and tool dependencies consecutively since each redirects to an ajaxian grid. This - # is not a problem in that Galaxy API. message = kwd.get( 'message', '' ) status = kwd.get( 'status', 'done' ) repository_id = kwd.get( 'id', None ) @@ -1160,15 +1158,27 @@ class AdminToolshed( AdminGalaxy ): message=message, status=status ) ) tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) + if kwd.get( 'repair_repository_button', False ): + repair_dict = kwd.get( 'repair_dict', None ) + if not repair_dict: + repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository ) + ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) + ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) + if ordered_tsr_ids and ordered_repo_info_dicts: + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='manage_repositories', + operation='repair', + ordered_tsr_ids=ordered_tsr_ids, + ordered_repo_info_dicts=ordered_repo_info_dicts ) ) + tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository ) ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) - if ordered_tsr_ids and ordered_repo_info_dicts: - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='manage_repositories', - operation='repair', - ordered_tsr_ids=ordered_tsr_ids, - ordered_repo_info_dicts=ordered_repo_info_dicts ) ) + return trans.fill_template( 'admin/tool_shed_repository/repair_repository.mako', + repository=tool_shed_repository, + repair_dict=repair_dict, + message=message, + status=status ) @web.json def repository_installation_status_updates( self, trans, ids=None, status_list=None ): diff --git a/templates/admin/tool_shed_repository/repair_repository.mako b/templates/admin/tool_shed_repository/repair_repository.mako new file mode 100644 index 00000000000..721d4da829f --- /dev/null +++ b/templates/admin/tool_shed_repository/repair_repository.mako @@ -0,0 +1,51 @@ +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="render_msg" /> +<%namespace file="/admin/tool_shed_repository/repository_actions_menu.mako" import="*" /> + +<%def name="stylesheets()"> + ${parent.stylesheets()} + ${h.css( "library" )} + + +<%def name="javascripts()"> + ${parent.javascripts()} + + +${render_galaxy_repository_actions( repository )} + +%if message: + ${render_msg( message, status )} +%endif + +
+ The following repositories will be inspected and repaired in the order listed to ensure each repository and all of it's tool dependencies are correctly + installed. Click Repair to inspect and repair these repositories. +
+ +
+
Repair tool shed repository '${repository.name}'
+

+
+ <% ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) %> +
Tool id: ${passed_test.tool_id | h}
+ + %for repo_info_dict in ordered_repo_info_dicts: + <% + for name, repo_info_tuple in repo_info_dict.items(): + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple + break + %> + + + + + + %endfor +
NameOwnerChangeset revision
${name | h}${repository_owner | h}${changeset_revision | h}
+
+
+ +
+ + + From 6a80507ea1b02705e2fae42cd02f56abf9fcef6b Mon Sep 17 00:00:00 2001 From: Adam Brenner Date: Thu, 27 Jun 2013 13:50:57 -0700 Subject: [PATCH 233/318] Replaced "binary" with "package" for eXpress according to my listsrv report of not picking up the correct environmental variables. http://dev.list.galaxyproject.org/Tool-Dependency-Check-for-eXpress-requirements-syntax-incorrect-td4660416.html Signed-off-by: Adam Brenner --- tools/ngs_rna/express_wrapper.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ngs_rna/express_wrapper.xml b/tools/ngs_rna/express_wrapper.xml index 5b429117158..04880a3217c 100644 --- a/tools/ngs_rna/express_wrapper.xml +++ b/tools/ngs_rna/express_wrapper.xml @@ -1,7 +1,7 @@ Quantify the abundances of a set of target sequences from sampled subsequences - eXpress + eXpress express --no-update-check @@ -122,4 +122,4 @@ eXpress requires two input files: - If you choose to use to calculate the covariance between targets and outputs, an other output would be : varcov.xprs. - \ No newline at end of file + From 85918b7a7a9acc011b1ab30e0f54df2828aa9001 Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Fri, 28 Jun 2013 15:26:27 -0400 Subject: [PATCH 234/318] Enhance multi-value select parsing to split on all whitespace; this fixes a bug that prevents multi-value selects from being used in workflows due to web browsers using \r\n for newline designation. --- lib/galaxy/tools/parameters/basic.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 1f8dacba407..b2f69731d15 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -703,7 +703,10 @@ class SelectToolParameter( ToolParameter ): value = None else: if not isinstance( value, list ): - value = value.split( "\n" ) + # Split on all whitespace. This not only provides flexibility + # in interpreting values but also is needed because many browsers + # use \r\n to separate lines. + value = value.split() return UnvalidatedValue( value ) legal_values = self.get_legal_values( trans, context ) if isinstance( value, list ): From 8cf8bc52f28c1bb6e8a7347fc23011e7c7a25b53 Mon Sep 17 00:00:00 2001 From: Aysam Guerler Date: Sun, 30 Jun 2013 08:34:19 +0200 Subject: [PATCH 235/318] Fix event handling, tooltip --- static/scripts/galaxy.frame.js | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/static/scripts/galaxy.frame.js b/static/scripts/galaxy.frame.js index 7a38d6f3af7..4aaa3cfdd27 100644 --- a/static/scripts/galaxy.frame.js +++ b/static/scripts/galaxy.frame.js @@ -91,8 +91,8 @@ var GalaxyFrameManager = Backbone.View.extend( $(this.el).append(this.frame_template_menu()); // load load button - $(this.el_header).append(this.frame_template_load()); - + $(this.el_header).append(this.frame_template_header()); + // // define shadow frame // @@ -121,7 +121,11 @@ var GalaxyFrameManager = Backbone.View.extend( // link events this.event_initialize(); - + + // add + $(".galaxy-frame-active").tooltip({title: "Enable/Disable Scratchbook"}); + $(".galaxy-frame-load").tooltip({title: "Show/Hide Scratchbook"}); + // catch window resize event var self = this; $(window).resize(function () @@ -162,7 +166,6 @@ var GalaxyFrameManager = Backbone.View.extend( {*/ this.events = { // global page events - 'mousedown' : 'event_frame_mouse_down', 'mousemove' : 'event_frame_mouse_move', 'mouseup' : 'event_frame_mouse_up', 'mouseleave' : 'event_frame_mouse_up', @@ -170,13 +173,14 @@ var GalaxyFrameManager = Backbone.View.extend( 'DOMMouseScroll' : 'event_panel_scroll', // events fixed to elements - 'mousedown .f-close' : 'event_frame_close', - 'mousedown .f-pin' : 'event_frame_lock', + 'mousedown .galaxy-frame' : 'event_frame_mouse_down', 'mousedown .galaxy-frame-active' : 'event_panel_active', 'mousedown .galaxy-frame-load' : 'event_panel_load', 'mousedown .galaxy-frame-background' : 'event_panel_load', 'mousedown .galaxy-frame-scroll-up' : 'event_panel_scroll_up', - 'mousedown .galaxy-frame-scroll-down' : 'event_panel_scroll_down' + 'mousedown .galaxy-frame-scroll-down' : 'event_panel_scroll_down', + 'mousedown .f-close' : 'event_frame_close', + 'mousedown .f-pin' : 'event_frame_lock' }; /*} else { this.events = { @@ -197,9 +201,6 @@ var GalaxyFrameManager = Backbone.View.extend( // drag start event_frame_mouse_down: function (e) { - // prevent text selection - e.preventDefault(); - // skip if event is already active if (this.event.type !== null) return; @@ -212,11 +213,14 @@ var GalaxyFrameManager = Backbone.View.extend( // check for resize event if ($(e.target).hasClass('f-resize')) this.event.type = 'resize'; - + // skip if no event has to be handled if (this.event.type === null) return; - + + // prevent text selection + e.preventDefault(); + // identify frame this.event.target = this.event_get_frame(e.target); @@ -1044,13 +1048,13 @@ var GalaxyFrameManager = Backbone.View.extend( }, // fill load button template - frame_template_load: function() + frame_template_header: function() { return '
' + '
0
' + '
' + - '
'+ - '
' + + '
' + + '
' + '
' + '
'; }, From 38de24198cad1e9eb9a9f876c5b5f081400ca734 Mon Sep 17 00:00:00 2001 From: Aysam Guerler Date: Sun, 30 Jun 2013 09:27:29 +0200 Subject: [PATCH 236/318] Scratchbook: Improve event handling --- static/scripts/galaxy.frame.js | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/static/scripts/galaxy.frame.js b/static/scripts/galaxy.frame.js index 4aaa3cfdd27..97033f1245b 100644 --- a/static/scripts/galaxy.frame.js +++ b/static/scripts/galaxy.frame.js @@ -375,6 +375,9 @@ var GalaxyFrameManager = Backbone.View.extend( if (this.event.type !== null) return; + // prevent text selection + e.preventDefault(); + // get frame var frame = this.event_get_frame(e.target); @@ -404,23 +407,29 @@ var GalaxyFrameManager = Backbone.View.extend( }, // show/hide panel - event_panel_load: function () + event_panel_load: function (e) { // check if (this.event.type !== null) return; + // prevent text selection + e.preventDefault(); + // load panel this.panel_show_hide(); }, // activate/disable panel - event_panel_active: function () + event_panel_active: function (e) { // check if (this.event.type !== null) return; + // prevent text selection + e.preventDefault(); + // load panel this.panel_active_disable(); }, @@ -432,6 +441,9 @@ var GalaxyFrameManager = Backbone.View.extend( if (this.event.type !== null || !this.visible) return; + // prevent text selection + e.preventDefault(); + // get wheel delta var delta = e.originalEvent.detail ? e.originalEvent.detail : e.originalEvent.wheelDelta / -3; @@ -440,23 +452,29 @@ var GalaxyFrameManager = Backbone.View.extend( }, // scroll up - event_panel_scroll_up: function() + event_panel_scroll_up: function(e) { // check if (this.event.type !== null) return; + // prevent text selection + e.preventDefault(); + // scroll up this.panel_scroll(-this.options.scroll); }, // scroll down - event_panel_scroll_down: function() + event_panel_scroll_down: function(e) { // check if (this.event.type !== null) return; + // prevent text selection + e.preventDefault(); + // scroll down this.panel_scroll(this.options.scroll); }, @@ -771,6 +789,9 @@ var GalaxyFrameManager = Backbone.View.extend( // adds and displays a new frame/window frame_new: function(options) { + // trigger mouse leave + $(document).trigger('mouseleave'); + // validate if (!this.active) { From 86f25f3bcf708127e117ddf2692900c8f760c695 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Sun, 30 Jun 2013 10:04:58 -0400 Subject: [PATCH 237/318] GCC2013 Advanced Tool and Data Source Configuration workshop: "The test framework uses tool_conf.xml.sample for reasons unknown and lost to time." Use tool_conf.xml for running functional tests, rather than tool_conf.xml.sample. --- buildbot_setup.sh | 1 + run_functional_tests.sh | 1 - scripts/functional_tests.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildbot_setup.sh b/buildbot_setup.sh index d4917e8c8e2..8eb2f73a372 100644 --- a/buildbot_setup.sh +++ b/buildbot_setup.sh @@ -65,6 +65,7 @@ LINKS=" " SAMPLES=" +tool_conf.xml.sample datatypes_conf.xml.sample universe_wsgi.ini.sample tool_data_table_conf.xml.sample diff --git a/run_functional_tests.sh b/run_functional_tests.sh index d4be604ff8b..b467974ef46 100755 --- a/run_functional_tests.sh +++ b/run_functional_tests.sh @@ -51,4 +51,3 @@ else fi echo "'run_functional_tests.sh help' for help" -echo "Note: galaxy test framework uses tool_conf.xml.sample, not tool_conf.xml" diff --git a/scripts/functional_tests.py b/scripts/functional_tests.py index a0a7abc183a..42f9f9299ae 100644 --- a/scripts/functional_tests.py +++ b/scripts/functional_tests.py @@ -129,7 +129,7 @@ def main(): # Exclude all files except test_toolbox.py. ignore_files = ( re.compile( r'^test_[adghlmsu]*' ), re.compile( r'^test_ta*' ) ) else: - tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', 'tool_conf.xml.sample' ) + tool_config_file = os.environ.get( 'GALAXY_TEST_TOOL_CONF', 'tool_conf.xml' ) galaxy_test_file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', default_galaxy_test_file_dir ) if not os.path.isabs( galaxy_test_file_dir ): galaxy_test_file_dir = os.path.join( os.getcwd(), galaxy_test_file_dir ) From 430303df5d92472d852b38e439351f6c5b614aa9 Mon Sep 17 00:00:00 2001 From: Aysam Guerler Date: Mon, 1 Jul 2013 01:29:04 +0200 Subject: [PATCH 238/318] scratchbook: hide on disable --- static/scripts/galaxy.frame.js | 28 ++++++++++++++++------------ static/style/blue/galaxy.frame.css | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/static/scripts/galaxy.frame.js b/static/scripts/galaxy.frame.js index 97033f1245b..be84b5a6ced 100644 --- a/static/scripts/galaxy.frame.js +++ b/static/scripts/galaxy.frame.js @@ -123,8 +123,8 @@ var GalaxyFrameManager = Backbone.View.extend( this.event_initialize(); // add - $(".galaxy-frame-active").tooltip({title: "Enable/Disable Scratchbook"}); - $(".galaxy-frame-load").tooltip({title: "Show/Hide Scratchbook"}); + //$(".galaxy-frame-active").tooltip({title: "Enable/Disable Scratchbook"}); + //$(".galaxy-frame-load").tooltip({title: "Show/Hide Scratchbook"}); // catch window resize event var self = this; @@ -218,7 +218,7 @@ var GalaxyFrameManager = Backbone.View.extend( if (this.event.type === null) return; - // prevent text selection + // prevent e.preventDefault(); // identify frame @@ -340,6 +340,9 @@ var GalaxyFrameManager = Backbone.View.extend( if (this.event.type !== null) return; + // prevent + e.preventDefault(); + // get frame var frame = this.event_get_frame(e.target); var self = this; @@ -375,7 +378,7 @@ var GalaxyFrameManager = Backbone.View.extend( if (this.event.type !== null) return; - // prevent text selection + // prevent e.preventDefault(); // get frame @@ -413,7 +416,7 @@ var GalaxyFrameManager = Backbone.View.extend( if (this.event.type !== null) return; - // prevent text selection + // prevent e.preventDefault(); // load panel @@ -427,7 +430,7 @@ var GalaxyFrameManager = Backbone.View.extend( if (this.event.type !== null) return; - // prevent text selection + // prevent e.preventDefault(); // load panel @@ -441,7 +444,7 @@ var GalaxyFrameManager = Backbone.View.extend( if (this.event.type !== null || !this.visible) return; - // prevent text selection + // prevent e.preventDefault(); // get wheel delta @@ -458,7 +461,7 @@ var GalaxyFrameManager = Backbone.View.extend( if (this.event.type !== null) return; - // prevent text selection + // prevent e.preventDefault(); // scroll up @@ -472,7 +475,7 @@ var GalaxyFrameManager = Backbone.View.extend( if (this.event.type !== null) return; - // prevent text selection + // prevent e.preventDefault(); // scroll down @@ -774,6 +777,10 @@ var GalaxyFrameManager = Backbone.View.extend( // untoggle $(".galaxy-frame-active .icon").removeClass("f-toggle"); + + // hide panel + if (this.visible) + this.panel_show_hide(); } else { // activate this.active = true; @@ -789,9 +796,6 @@ var GalaxyFrameManager = Backbone.View.extend( // adds and displays a new frame/window frame_new: function(options) { - // trigger mouse leave - $(document).trigger('mouseleave'); - // validate if (!this.active) { diff --git a/static/style/blue/galaxy.frame.css b/static/style/blue/galaxy.frame.css index f2b0cb829e4..3e6c36c3d16 100644 --- a/static/style/blue/galaxy.frame.css +++ b/static/style/blue/galaxy.frame.css @@ -95,7 +95,7 @@ z-index : 34010; position : absolute; top : 6px; - right : 165px; + right : 170px; cursor : pointer; color : #BCC800; } From 6bd19c520fb9b1fe515700fe403542310592775e Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Mon, 1 Jul 2013 05:57:51 -0400 Subject: [PATCH 239/318] Fix converter imports -- resolves failing unit tests --- .../interval_to_interval_index_converter.py | 18 ++++++++---------- .../pileup_to_interval_index_converter.py | 16 +++++++--------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py index 130eec6feea..8d1b7653cb0 100644 --- a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py @@ -11,15 +11,13 @@ usage: %prog in_file out_file from __future__ import division -import sys, fileinput, optparse +import optparse from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -from galaxy.visualization.tracks.summary import * -from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed +eggs.require( "bx-python" ) from bx.interval_index_file import Indexes def main(): - + # Read options, args. parser = optparse.OptionParser() parser.add_option( '-c', '--chr-col', type='int', dest='chrom_col', default=1 ) @@ -27,12 +25,12 @@ def main(): parser.add_option( '-e', '--end-col', type='int', dest='end_col', default=3 ) (options, args) = parser.parse_args() input_fname, output_fname = args - + # Make column indices 0-based. options.chrom_col -= 1 options.start_col -= 1 options.end_col -= 1 - + # Do conversion. index = Indexes() offset = 0 @@ -46,9 +44,9 @@ def main(): chrom_end = int( feature[ options.end_col ] ) index.add( chrom, chrom_start, chrom_end, offset ) offset += len(line) - + index.write( open(output_fname, "w") ) -if __name__ == "__main__": +if __name__ == "__main__": main() - \ No newline at end of file + diff --git a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py index 21c652ba144..88e76b40f33 100644 --- a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py +++ b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py @@ -8,20 +8,18 @@ usage: %prog in_file out_file from __future__ import division -import sys, fileinput, optparse +import optparse from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -from galaxy.visualization.tracks.summary import * -from galaxy.datatypes.util.gff_util import convert_gff_coords_to_bed +eggs.require( "bx-python" ) from bx.interval_index_file import Indexes def main(): - + # Read options, args. parser = optparse.OptionParser() (options, args) = parser.parse_args() input_fname, output_fname = args - + # Do conversion. index = Indexes() offset = 0 @@ -31,9 +29,9 @@ def main(): start = int( start ) - 1 index.add( chrom, start, start + 1, offset ) offset += len( line ) - + index.write( open(output_fname, "w") ) -if __name__ == "__main__": +if __name__ == "__main__": main() - \ No newline at end of file + From b096a23b98f97757ef9753aa2d10a89e04f016ef Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Mon, 1 Jul 2013 09:46:09 -0400 Subject: [PATCH 240/318] Allow spaces and commas in Cuffdiff labels. --- tools/ngs_rna/cuffdiff_wrapper.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/ngs_rna/cuffdiff_wrapper.xml b/tools/ngs_rna/cuffdiff_wrapper.xml index c9ba78f7404..b4fe1bc6859 100644 --- a/tools/ngs_rna/cuffdiff_wrapper.xml +++ b/tools/ngs_rna/cuffdiff_wrapper.xml @@ -59,7 +59,9 @@ ## Replicates. --labels #for $group in $group_analysis.groups - "${group.group}" + ## Cuffdiff uses commas as delimiters, so replace them with underscores to avoid + ## parsing problems. + "${group.group.replace(',', '_')}" #end for --files #for $group in $group_analysis.groups @@ -80,7 +82,7 @@ - + From b1d3a2475d715d278763fb652c13541046d92e0a Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Mon, 1 Jul 2013 12:37:44 -0400 Subject: [PATCH 241/318] Galaxy pages: (a) move JS code out of mako template and into galaxy.pages.js; (b) make WYMEditor work with Chrome and jQuery 1.9+; (c) fix bug that showed meaningless text when inserting a link. Pack scripts. --- static/scripts/galaxy.pages.js | 708 ++++++++++++++++ .../scripts/libs/jquery/jquery.wymeditor.js | 3 +- static/scripts/packed/galaxy.frame.js | 2 +- static/scripts/packed/galaxy.pages.js | 1 + .../packed/libs/jquery/jquery.wymeditor.js | 2 +- static/scripts/packed/mvc/data.js | 2 +- static/scripts/packed/utils/galaxy.css.js | 2 +- templates/webapps/galaxy/page/editor.mako | 755 +----------------- 8 files changed, 728 insertions(+), 747 deletions(-) create mode 100644 static/scripts/galaxy.pages.js create mode 100644 static/scripts/packed/galaxy.pages.js diff --git a/static/scripts/galaxy.pages.js b/static/scripts/galaxy.pages.js new file mode 100644 index 00000000000..7960d26345b --- /dev/null +++ b/static/scripts/galaxy.pages.js @@ -0,0 +1,708 @@ +// Useful Galaxy stuff. +var Galaxy = +{ + // Item types. + ITEM_HISTORY : "item_history", + ITEM_DATASET : "item_dataset", + ITEM_WORKFLOW : "item_workflow", + ITEM_PAGE : "item_page", + ITEM_VISUALIZATION : "item_visualization", + + // Link dialogs. + DIALOG_HISTORY_LINK : "link_history", + DIALOG_DATASET_LINK : "link_dataset", + DIALOG_WORKFLOW_LINK : "link_workflow", + DIALOG_PAGE_LINK : "link_page", + DIALOG_VISUALIZATION_LINK : "link_visualization", + + // Embed dialogs. + DIALOG_EMBED_HISTORY : "embed_history", + DIALOG_EMBED_DATASET : "embed_dataset", + DIALOG_EMBED_WORKFLOW : "embed_workflow", + DIALOG_EMBED_PAGE : "embed_page", + DIALOG_EMBED_VISUALIZATION : "embed_visualization", + + // Annotation dialogs. + DIALOG_HISTORY_ANNOTATE : "history_annotate", +}; + +// Initialize Galaxy elements. +function init_galaxy_elts(wym) +{ + // Set up events to make annotation easy. + $('.annotation', wym._doc.body).each( function() + { + $(this).click( function() { + // Works in Safari, not in Firefox. + var range = wym._doc.createRange(); + range.selectNodeContents( this ); + var selection = window.getSelection(); + selection.removeAllRanges(); + selection.addRange(range); + var t = ""; + }); + }); + +}; + +// Based on the dialog type, return a dictionary of information about an item +function get_item_info( dialog_type ) +{ + var + item_singular, + item_plural, + item_controller; + switch( dialog_type ) { + case( Galaxy.ITEM_HISTORY ): + item_singular = "History"; + item_plural = "Histories"; + item_controller = "history"; + item_class = "History"; + break; + case( Galaxy.ITEM_DATASET ): + item_singular = "Dataset"; + item_plural = "Datasets"; + item_controller = "dataset"; + item_class = "HistoryDatasetAssociation"; + break; + case( Galaxy.ITEM_WORKFLOW ): + item_singular = "Workflow"; + item_plural = "Workflows"; + item_controller = "workflow"; + item_class = "StoredWorkflow"; + break; + case( Galaxy.ITEM_PAGE ): + item_singular = "Page"; + item_plural = "Pages"; + item_controller = "page"; + item_class = "Page"; + break; + case( Galaxy.ITEM_VISUALIZATION ): + item_singular = "Visualization"; + item_plural = "Visualizations"; + item_controller = "visualization"; + item_class = "Visualization"; + break; + } + + // Build ajax URL that lists items for selection. + var item_list_action = "list_" + item_plural.toLowerCase() + "_for_selection"; + var ajax_url = list_objects_url.replace( "LIST_ACTION", item_list_action ); + + // Set up and return dict. + return { + singular : item_singular, + plural : item_plural, + controller : item_controller, + iclass : item_class, + list_ajax_url : ajax_url + }; +}; + +// Make an item importable. +function make_item_importable( item_controller, item_id, item_type ) +{ + ajax_url = set_accessible_url.replace( "ITEM_CONTROLLER", item_controller ); + $.ajax({ + type: "POST", + url: ajax_url, + data: { id: item_id, accessible: 'True' }, + error: function() { alert("Making " + item_type + " accessible failed"); } + }); +}; + +// Completely replace WYM's dialog handling +WYMeditor.editor.prototype.dialog = function( dialogType, dialogFeatures, bodyHtml ) { + + var wym = this; + var sStamp = wym.uniqueStamp(); + var selected = wym.selected(); + + // Swap out URL attribute for id/name attribute in link creation to enable anchor creation in page. + function set_link_id() + { + // When "set link id" link clicked, update UI. + $('#set_link_id').click( function() + { + // Set label. + $("#link_attribute_label").text("ID/Name"); + + // Set input elt class, value. + var attribute_input = $(".wym_href"); + attribute_input.addClass("wym_id").removeClass("wym_href"); + if (selected) + attribute_input.val( $(selected).attr('id') ); + + // Remove link. + $(this).remove(); + }); + } + + // LINK DIALOG + if ( dialogType == WYMeditor.DIALOG_LINK ) { + if(selected) { + $(wym._options.hrefSelector).val($(selected).attr(WYMeditor.HREF)); + $(wym._options.srcSelector).val($(selected).attr(WYMeditor.SRC)); + $(wym._options.titleSelector).val($(selected).attr(WYMeditor.TITLE)); + $(wym._options.altSelector).val($(selected).attr(WYMeditor.ALT)); + } + // Get current URL, title. + var curURL, curTitle; + if (selected) + { + curURL = $(selected).attr("href"); + if (curURL == undefined) + curURL = ""; + curTitle = $(selected).attr("title"); + if (curTitle == undefined) + curTitle = ""; + } + show_modal( + "Create Link", + "
" + + "

", + { + "Make link": function() { + // Get URL, name/title. + var sUrl = $(wym._options.hrefSelector).val(), + sName = $(".wym_title").val(); + + if (sUrl && sName) { + // Create link. + wym._exec(WYMeditor.CREATE_LINK, sStamp); + + // Set link attributes. + var link = $("a[href=" + sStamp + "]", wym._doc.body); + link.attr(WYMeditor.HREF, sUrl) + .attr(WYMeditor.TITLE, $(wym._options.titleSelector).val()) + .attr("id", sName); + + // If link's text is default (wym-...), change it to the title. + if (link.text().indexOf('wym-') === 0) { + link.text(sName); + } + } + hide_modal(); + }, + "Cancel": function() { + hide_modal(); + } + }, + {}, + set_link_id + ); + } + + // IMAGE DIALOG + if ( dialogType == WYMeditor.DIALOG_IMAGE ) { + if(wym._selected_image) { + $(wym._options.dialogImageSelector + " " + wym._options.srcSelector) + .val($(wym._selected_image).attr(WYMeditor.SRC)); + $(wym._options.dialogImageSelector + " " + wym._options.titleSelector) + .val($(wym._selected_image).attr(WYMeditor.TITLE)); + $(wym._options.dialogImageSelector + " " + wym._options.altSelector) + .val($(wym._selected_image).attr(WYMeditor.ALT)); + } + show_modal( + "Image", + "
" + + "
" + + "" + + "
" + + "
" + + "
" + + "" + + "
" + + "
" + + "
" + + "" + + "
", + { + "Insert": function() { + var sUrl = $(wym._options.srcSelector).val(); + if(sUrl.length > 0) { + wym._exec(WYMeditor.INSERT_IMAGE, sStamp); + $("img[src$=" + sStamp + "]", wym._doc.body) + .attr(WYMeditor.SRC, sUrl) + .attr(WYMeditor.TITLE, $(wym._options.titleSelector).val()) + .attr(WYMeditor.ALT, $(wym._options.altSelector).val()); + } + hide_modal(); + }, + "Cancel": function() { + hide_modal(); + } + } + ); + return; + } + + // TABLE DIALOG + if ( dialogType == WYMeditor.DIALOG_TABLE ) { + show_modal( + "Table", + "
" + + "
" + + "" + + "
" + + "
" + + "
" + + "" + + "
" + + "
" + + "" + + "" + + "
" + + "
" + + "" + + "" + + "
", + { + "Insert": function() { + var iRows = $(wym._options.rowsSelector).val(); + var iCols = $(wym._options.colsSelector).val(); + + if(iRows > 0 && iCols > 0) { + + var table = wym._doc.createElement(WYMeditor.TABLE); + var newRow = null; + var newCol = null; + + var sCaption = $(wym._options.captionSelector).val(); + + //we create the caption + var newCaption = table.createCaption(); + newCaption.innerHTML = sCaption; + + //we create the rows and cells + for(x=0; x" + + "Make the selected " + item_info.plural.toLowerCase() + " accessible so that they can viewed by everyone.
" + , + { + "Insert": function() + { + // Make selected items accessible (importable) ? + var make_importable = false; + if ( $('#make-importable:checked').val() !== null ) + make_importable = true; + + // Insert links to history for each checked item. + var item_ids = new Array(); + $('input[name=id]:checked').each(function() { + var item_id = $(this).val(); + + // Make item importable? + if (make_importable) + make_item_importable(item_info.controller, item_id, item_info.singular); + + // Insert link(s) to item(s). This is done by getting item info and then manipulating wym. + url_template = get_name_and_link_url + item_id; + ajax_url = url_template.replace( "ITEM_CONTROLLER", item_info.controller); + $.getJSON( ajax_url, function( returned_item_info ) { + // Get link text. + wym._exec(WYMeditor.CREATE_LINK, sStamp); + var link_text = $("a[href=" + sStamp + "]", wym._doc.body).text(); + + // Insert link: need to do different actions depending on link text. + if ( + link_text == "" // Firefox. + || + link_text == sStamp // Safari + ) + { + // User selected no text; create link from scratch and use default text. + wym.insert("" + item_info.singular + " '" + returned_item_info.name + "'"); + } + else + { + // Link created from selected text; add href and title. + $("a[href=" + sStamp + "]", wym._doc.body).attr(WYMeditor.HREF, returned_item_info.link).attr(WYMeditor.TITLE, item_info.singular + item_id); + } + }); + }); + + hide_modal(); + }, + "Cancel": function() + { + hide_modal(); + } + } + ); + } + }); + } + // EMBED GALAXY OBJECT DIALOGS + if ( dialogType == Galaxy.DIALOG_EMBED_HISTORY || dialogType == Galaxy.DIALOG_EMBED_DATASET || dialogType == Galaxy.DIALOG_EMBED_WORKFLOW || dialogType == Galaxy.DIALOG_EMBED_PAGE || dialogType == Galaxy.DIALOG_EMBED_VISUALIZATION ) { + // Based on item type, set useful vars. + var item_info; + switch(dialogType) + { + case(Galaxy.DIALOG_EMBED_HISTORY): + item_info = get_item_info(Galaxy.ITEM_HISTORY); + break; + case(Galaxy.DIALOG_EMBED_DATASET): + item_info = get_item_info(Galaxy.ITEM_DATASET); + break; + case(Galaxy.DIALOG_EMBED_WORKFLOW): + item_info = get_item_info(Galaxy.ITEM_WORKFLOW); + break; + case(Galaxy.DIALOG_EMBED_PAGE): + item_info = get_item_info(Galaxy.ITEM_PAGE); + break; + case(Galaxy.DIALOG_EMBED_VISUALIZATION): + item_info = get_item_info(Galaxy.ITEM_VISUALIZATION); + break; + } + + $.ajax( + { + url: item_info.list_ajax_url, + data: {}, + error: function() { alert( "Failed to list " + item_info.plural.toLowerCase() + " for selection"); }, + success: function(list_html) + { + // Can make histories, workflows importable; cannot make datasets importable. + if (dialogType == Galaxy.DIALOG_EMBED_HISTORY || dialogType == Galaxy.DIALOG_EMBED_WORKFLOW + || dialogType == Galaxy.DIALOG_EMBED_VISUALIZATION) + list_html = list_html + "
" + + "Make the selected " + item_info.plural.toLowerCase() + " accessible so that they can viewed by everyone.
"; + show_modal( + "Embed " + item_info.plural, + list_html, + { + "Embed": function() + { + // Make selected items accessible (importable) ? + var make_importable = false; + if ( $('#make-importable:checked').val() != null ) + make_importable = true; + + $('input[name=id]:checked').each(function() { + // Get item ID and name. + var item_id = $(this).val(); + // Use ':first' because there are many labels in table; the first one is the item name. + var item_name = $("label[for='" + item_id + "']:first").text(); + + if (make_importable) + make_item_importable(item_info.controller, item_id, item_info.singular); + + // Embedded item HTML; item class is embedded in div container classes; this is necessary because the editor strips + // all non-standard attributes when it returns its content (e.g. it will not return an element attribute of the form + // item_class='History'). + var item_elt_id = item_info.iclass + "-" + item_id; + var item_embed_html = + "

\ +

Embedded Galaxy " + item_info.singular + " '" + item_name + "'

\ +

\ + [Do not edit this block; Galaxy will fill it in with the annotated " + + item_info.singular.toLowerCase() + " when it is displayed.] \ +

\ +

"; + + // Insert embedded item into document. + wym.insert(" "); // Needed to prevent insertion from occurring in child element in webkit browsers. + wym.insert(item_embed_html); + + // TODO: can we fix this? + // Due to oddities of wym.insert() [likely due to inserting a
and/or a complete paragraph], an + // empty paragraph (or two!) may be included either before an embedded item. Remove these paragraphs. + $("#" + item_elt_id, wym._doc.body).each( function() { + // Remove previous empty paragraphs. + var removing = true; + while (removing) + { + var prev_elt = $(this).prev(); + if ( prev_elt.length != 0 && jQuery.trim(prev_elt.text()) == "" ) + prev_elt.remove(); + else + removing = false; + } + }); + + }); + hide_modal(); + }, + "Cancel": function() + { + hide_modal(); + } + } + ); + } + }); + } + + // ANNOTATE HISTORY DIALOG + if ( dialogType == Galaxy.DIALOG_ANNOTATE_HISTORY ) { + $.ajax( + { + url: list_histories_for_selection_url, + data: {}, + error: function() { alert( "Grid refresh failed" ) }, + success: function(table_html) + { + show_modal( + "Insert Link to History", + table_html, + { + "Annotate": function() + { + // Insert links to history for each checked item. + var item_ids = new Array(); + $('input[name=id]:checked').each(function() { + var item_id = $(this).val(); + + // Get annotation table for history. + $.ajax( + { + url: get_history_annotation_table_url, + data: { id : item_id }, + error: function() { alert( "Grid refresh failed" ) }, + success: function(result) + { + // Insert into document. + wym.insert(result); + + init_galaxy_elts(wym); + + } + }); + }); + + hide_modal(); + }, + "Cancel": function() + { + hide_modal(); + } + } + ); + } + }); + } +}; + +$(function(){ + // Generic error handling + $(document).ajaxError( function ( e, x ) { + // console.log( e, x ); + var message = x.responseText || x.statusText || "Could not connect to server"; + show_modal( "Server error", message, { "Ignore error" : hide_modal } ); + return false; + }); + // Create editor + $("[name=page_content]").wymeditor( { + skin: 'galaxy', + basePath: editor_base_path, + iframeBasePath: iframe_base_path, + boxHtml: "" + + "" + + "
" + + WYMeditor.TOOLS + + "
" + + "
" + // + WYMeditor.HTML + + WYMeditor.IFRAME + + WYMeditor.STATUS + + "
" + + "" + + "
", + toolsItems: [ + {'name': 'Bold', 'title': 'Strong', 'css': 'wym_tools_strong'}, + {'name': 'Italic', 'title': 'Emphasis', 'css': 'wym_tools_emphasis'}, + {'name': 'Superscript', 'title': 'Superscript', 'css': 'wym_tools_superscript'}, + {'name': 'Subscript', 'title': 'Subscript', 'css': 'wym_tools_subscript'}, + {'name': 'InsertOrderedList', 'title': 'Ordered_List', 'css': 'wym_tools_ordered_list'}, + {'name': 'InsertUnorderedList', 'title': 'Unordered_List', 'css': 'wym_tools_unordered_list'}, + {'name': 'Indent', 'title': 'Indent', 'css': 'wym_tools_indent'}, + {'name': 'Outdent', 'title': 'Outdent', 'css': 'wym_tools_outdent'}, + {'name': 'Undo', 'title': 'Undo', 'css': 'wym_tools_undo'}, + {'name': 'Redo', 'title': 'Redo', 'css': 'wym_tools_redo'}, + {'name': 'CreateLink', 'title': 'Link', 'css': 'wym_tools_link'}, + {'name': 'Unlink', 'title': 'Unlink', 'css': 'wym_tools_unlink'}, + {'name': 'InsertImage', 'title': 'Image', 'css': 'wym_tools_image'}, + {'name': 'InsertTable', 'title': 'Table', 'css': 'wym_tools_table'}, + ] + }); + // Get the editor object + var editor = $.wymeditors(0); + var save = function ( callback ) { + show_modal( "Saving page", "progress" ); + + // Do save. + $.ajax( { + url: save_url, + type: "POST", + data: { + id: page_id, + content: editor.xhtml(), + annotations: JSON.stringify(new Object()), + // annotations: JSON.stringify(annotations), + "_": "true" + }, + success: function() { + callback(); + } + }); + } + // Save button + $("#save-button").click( function() { + save( function() { hide_modal(); } ) + }); + // Close button + $("#close-button").click(function() { + // var new_content = editor.xhtml(); + // var changed = ( initial_content != new_content ); + var changed = false; + if ( changed ) { + var do_close = function() { + window.onbeforeunload = undefined; + window.document.location = page_list_url; + }; + show_modal( "Close editor", + "There are unsaved changes to your page which will be lost.", + { + "Cancel" : hide_modal, + "Save Changes" : function() { + save( do_close ); + } + }, { + "Don't Save": do_close + } ); + } else { + window.document.location = page_list_url; + } + }); + + // Initialize galaxy elements. + //init_galaxy_elts(editor); + + // + // Containers, Galaxy style + // + var containers_menu = $("
Paragraph type
"); + $(".wym_area_top").append( containers_menu ); + + // Add menu options. + var items = {} + $.each( editor._options.containersItems, function( k, v ) { + var tagname = v.name; + items[ v.title.replace( '_', ' ' ) ] = function() { editor.container( tagname ) } + }); + make_popupmenu( containers_menu, items); + + // + // Create 'Insert Link to Galaxy Object' menu. + // + + // Add menu button. + var insert_link_menu_button = $("").addClass('galaxy-page-editor-button'); + $(".wym_area_top").append(insert_link_menu_button); + + // Add menu options. + make_popupmenu( insert_link_menu_button, { + "Insert History Link": function() { + editor.dialog(Galaxy.DIALOG_HISTORY_LINK); + }, + "Insert Dataset Link": function() { + editor.dialog(Galaxy.DIALOG_DATASET_LINK); + }, + "Insert Workflow Link": function() { + editor.dialog(Galaxy.DIALOG_WORKFLOW_LINK); + }, + "Insert Page Link": function() { + editor.dialog(Galaxy.DIALOG_PAGE_LINK); + }, + "Insert Visualization Link": function() { + editor.dialog(Galaxy.DIALOG_VISUALIZATION_LINK); + }, + }); + + // + // Create 'Embed Galaxy Object' menu. + // + + // Add menu button. + var embed_object_button = $("").addClass('galaxy-page-editor-button'); + $(".wym_area_top").append(embed_object_button); + + // Add menu options. + make_popupmenu( embed_object_button, { + "Embed History": function() { + editor.dialog(Galaxy.DIALOG_EMBED_HISTORY); + }, + "Embed Dataset": function() { + editor.dialog(Galaxy.DIALOG_EMBED_DATASET); + }, + "Embed Workflow": function() { + editor.dialog(Galaxy.DIALOG_EMBED_WORKFLOW); + }, + "Embed Visualization": function() { + editor.dialog(Galaxy.DIALOG_EMBED_VISUALIZATION); + }, + //"Embed Page": function() { + // editor.dialog(Galaxy.DIALOG_EMBED_PAGE); + //} + }); +}); \ No newline at end of file diff --git a/static/scripts/libs/jquery/jquery.wymeditor.js b/static/scripts/libs/jquery/jquery.wymeditor.js index 2b0d894b975..e906032d00e 100644 --- a/static/scripts/libs/jquery/jquery.wymeditor.js +++ b/static/scripts/libs/jquery/jquery.wymeditor.js @@ -713,7 +713,8 @@ WYMeditor.editor.prototype.init = function() { else if (jQuery.browser.opera) { var WymClass = new WYMeditor.WymClassOpera(this); } - else if (jQuery.browser.safari) { + // Galaxy HACK: add Chrome to browser detection; this is fixed in later versions of WYMEditor. + else if (jQuery.browser.safari || jQuery.browser.chrome) { var WymClass = new WYMeditor.WymClassSafari(this); } diff --git a/static/scripts/packed/galaxy.frame.js b/static/scripts/packed/galaxy.frame.js index 1e97d2b5951..6ab83f16be6 100644 --- a/static/scripts/packed/galaxy.frame.js +++ b/static/scripts/packed/galaxy.frame.js @@ -1 +1 @@ -define(["utils/galaxy.css","libs/backbone/backbone-relational"],function(b){b.load_file("/static/style/galaxy.frame.css");b.load_file("/static/style/base.css");var a=Backbone.View.extend({el:"#everything",el_header:"#masthead",options:{frame:{cols:6,rows:3},rows:1000,cell:130,margin:5,scroll:5,top_min:40,frame_max:10},cols:0,top:0,top_max:0,frame_counter:0,frame_counter_id:0,frame_list:[],galaxy_frame_shadow:null,visible:false,active:false,initialize:function(d){if(d){this.options=_.defaults(d,this.options)}this.top=this.top_max=this.options.top_min;$(this.el).append(this.frame_template_background());$(this.el).append(this.frame_template_menu());$(this.el_header).append(this.frame_template_load());var e="#galaxy-frame-shadow";$(this.el).append(this.frame_template_shadow(e.substring(1)));this.galaxy_frame_shadow={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};this.frame_resize(this.galaxy_frame_shadow,{width:0,height:0});this.frame_list[e]=this.galaxy_frame_shadow;this.panel_refresh();this.event_initialize();var c=this;$(window).resize(function(){c.panel_refresh()});window.onbeforeunload=function(){if(c.frame_counter>0){return"You opened "+c.frame_counter+" frame(s) which will be lost."}}},is_mobile:function(){return navigator.userAgent.match(/mobile|(iPad)|(iPhone)|(iPod)|(android)|(webOS)/i)},event:{type:null,target:null,xy:null},event_initialize:function(){this.events={mousedown:"event_frame_mouse_down","mousedown .f-close":"event_frame_close","mousedown .f-pin":"event_frame_lock",mousemove:"event_frame_mouse_move",mouseup:"event_frame_mouse_up",mouseleave:"event_frame_mouse_up","mousedown .galaxy-frame-active":"event_panel_active","mousedown .galaxy-frame-load":"event_panel_load","mousedown .galaxy-frame-background":"event_panel_load",mousewheel:"event_panel_scroll",DOMMouseScroll:"event_panel_scroll","mousedown .galaxy-frame-scroll-up":"event_panel_scroll_up","mousedown .galaxy-frame-scroll-down":"event_panel_scroll_down"};this.delegateEvents(this.events)},event_frame_mouse_down:function(c){c.preventDefault();if(this.event.type!==null){return}if($(c.target).hasClass("f-header")||$(c.target).hasClass("f-title")){this.event.type="drag"}if($(c.target).hasClass("f-resize")){this.event.type="resize"}if(this.event.type===null){return}this.event.target=this.event_get_frame(c.target);if(this.event.target.grid_lock){this.event.type=null;return}this.event.xy={x:c.originalEvent.pageX,y:c.originalEvent.pageY};this.frame_drag_start(this.event.target)},event_frame_mouse_move:function(i){if(this.event.type!="drag"&&this.event.type!="resize"){return}var g={x:i.originalEvent.pageX,y:i.originalEvent.pageY};var d={x:g.x-this.event.xy.x,y:g.y-this.event.xy.y};this.event.xy=g;var h=this.frame_screen(this.event.target);if(this.event.type=="resize"){h.width+=d.x;h.height+=d.y;var f=this.options.cell-this.options.margin-1;h.width=Math.max(h.width,f);h.height=Math.max(h.height,f);this.frame_resize(this.event.target,h);h.width=this.to_grid_coord("width",h.width)+1;h.height=this.to_grid_coord("height",h.height)+1;h.width=this.to_pixel_coord("width",h.width);h.height=this.to_pixel_coord("height",h.height);this.frame_resize(this.galaxy_frame_shadow,h);this.frame_insert(this.galaxy_frame_shadow,{top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)})}if(this.event.type=="drag"){h.left+=d.x;h.top+=d.y;this.frame_offset(this.event.target,h);var c={top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)};if(c.left!==0){c.left++}this.frame_insert(this.galaxy_frame_shadow,c)}},event_frame_mouse_up:function(c){if(this.event.type!="drag"&&this.event.type!="resize"){return}this.frame_drag_stop(this.event.target);this.event.type=null},event_frame_close:function(d){if(this.event.type!==null){return}var f=this.event_get_frame(d.target);var c=this;$(f.id).fadeOut("fast",function(){$(f.id).remove();delete c.frame_list[f.id];c.frame_counter--;c.panel_refresh(true);c.panel_animation_complete();if(c.visible&&c.frame_counter==0){c.panel_show_hide()}})},event_frame_lock:function(c){if(this.event.type!==null){return}var d=this.event_get_frame(c.target);if(d.grid_lock){d.grid_lock=false;$(d.id).find(".f-pin").removeClass("f-toggle");$(d.id).find(".f-header").removeClass("f-not-allowed");$(d.id).find(".f-title").removeClass("f-not-allowed");$(d.id).find(".f-resize").show();$(d.id).find(".f-close").show()}else{d.grid_lock=true;$(d.id).find(".f-pin").addClass("f-toggle");$(d.id).find(".f-header").addClass("f-not-allowed");$(d.id).find(".f-title").addClass("f-not-allowed");$(d.id).find(".f-resize").hide();$(d.id).find(".f-close").hide()}},event_panel_load:function(){if(this.event.type!==null){return}this.panel_show_hide()},event_panel_active:function(){if(this.event.type!==null){return}this.panel_active_disable()},event_panel_scroll:function(c){var d=c.originalEvent.detail?c.originalEvent.detail:c.originalEvent.wheelDelta/-3;this.panel_scroll(d)},event_panel_scroll_up:function(){this.panel_scroll(-this.options.scroll)},event_panel_scroll_down:function(){this.panel_scroll(this.options.scroll)},event_get_frame:function(c){return this.frame_list["#"+$(c).closest(".galaxy-frame").attr("id")]},frame_drag_start:function(d){this.frame_focus(d,true);var c=this.frame_screen(d);this.frame_resize(this.galaxy_frame_shadow,c);this.frame_grid(this.galaxy_frame_shadow,d.grid_location);d.grid_location=null;$(this.galaxy_frame_shadow.id).show();$(".f-cover").show()},frame_drag_stop:function(d){this.frame_focus(d,false);var c=this.frame_screen(this.galaxy_frame_shadow);this.frame_resize(d,c);this.frame_grid(d,this.galaxy_frame_shadow.grid_location,true);this.galaxy_frame_shadow.grid_location=null;$(this.galaxy_frame_shadow.id).hide();$(".f-cover").hide();this.panel_animation_complete()},to_grid_coord:function(e,d){var c=(e=="width"||e=="height")?1:-1;if(e=="top"){d-=this.top}return parseInt((d+c*this.options.margin)/this.options.cell,10)},to_pixel_coord:function(e,f){var c=(e=="width"||e=="height")?1:-1;var d=(f*this.options.cell)-c*this.options.margin;if(e=="top"){d+=this.top}return d},to_grid:function(c){return{top:this.to_grid_coord("top",c.top),left:this.to_grid_coord("left",c.left),width:this.to_grid_coord("width",c.width),height:this.to_grid_coord("height",c.height)}},to_pixel:function(c){return{top:this.to_pixel_coord("top",c.top),left:this.to_pixel_coord("left",c.left),width:this.to_pixel_coord("width",c.width),height:this.to_pixel_coord("height",c.height)}},is_collision:function(e){function c(h,g){return !(h.left>g.left+g.width-1||h.left+h.width-1g.top+g.height-1||h.top+h.height-1this.frame_counter){this.top=this.options.top_min;d.width=this.to_pixel_coord("width",this.options.frame.cols);d.height=this.to_pixel_coord("height",this.options.frame.rows);$(this.el).append(this.frame_template(e.substring(1),d.title,d.type,d.content));var f={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};this.frame_counter++;this.frame_list[e]=f;this.frame_resize(f,{width:d.width,height:d.height});this.frame_insert(f,{top:0,left:0},true);if(!this.visible){this.panel_show_hide()}}else{alert("You have reached the maximum number of allowed frames ("+this.options.frame_max+").")}},frame_insert:function(j,c,e){var d=[];if(j){j.grid_location=null;d.push([j,this.location_rank(c)])}var g=null;for(g in this.frame_list){var h=this.frame_list[g];if(h.grid_location!==null&&!h.grid_lock){h.grid_location=null;d.push([h,h.grid_rank])}}d.sort(function(k,f){var m=k[1];var l=f[1];return ml?1:0)});for(g=0;g'}return'
'+e+'
'+d+'
'},frame_template_shadow:function(c){return'
'},frame_template_background:function(){return'
'},frame_template_load:function(){return'
0
'},frame_template_menu:function(){return'
'}});return{GalaxyFrameManager:a}}); \ No newline at end of file +define(["utils/galaxy.css","libs/backbone/backbone-relational"],function(b){var a=Backbone.View.extend({el:"#everything",el_header:"#masthead",options:{frame:{cols:6,rows:3},rows:1000,cell:130,margin:5,scroll:5,top_min:40,frame_max:10},cols:0,top:0,top_max:0,frame_counter:0,frame_counter_id:0,frame_list:[],galaxy_frame_shadow:null,visible:false,active:false,initialize:function(d){b.load_file(d.url.styles+"/galaxy.frame.css");if(d){this.options=_.defaults(d,this.options)}this.top=this.top_max=this.options.top_min;$(this.el).append(this.frame_template_background());$(this.el).append(this.frame_template_menu());$(this.el_header).append(this.frame_template_header());var e="#galaxy-frame-shadow";$(this.el).append(this.frame_template_shadow(e.substring(1)));this.galaxy_frame_shadow={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};this.frame_resize(this.galaxy_frame_shadow,{width:0,height:0});this.frame_list[e]=this.galaxy_frame_shadow;this.panel_refresh();this.event_initialize();var c=this;$(window).resize(function(){c.panel_refresh()});window.onbeforeunload=function(){if(c.frame_counter>0){return"You opened "+c.frame_counter+" frame(s) which will be lost."}}},is_mobile:function(){return navigator.userAgent.match(/mobile|(iPad)|(iPhone)|(iPod)|(android)|(webOS)/i)},event:{type:null,target:null,xy:null},event_initialize:function(){this.events={mousemove:"event_frame_mouse_move",mouseup:"event_frame_mouse_up",mouseleave:"event_frame_mouse_up",mousewheel:"event_panel_scroll",DOMMouseScroll:"event_panel_scroll","mousedown .galaxy-frame":"event_frame_mouse_down","mousedown .galaxy-frame-active":"event_panel_active","mousedown .galaxy-frame-load":"event_panel_load","mousedown .galaxy-frame-background":"event_panel_load","mousedown .galaxy-frame-scroll-up":"event_panel_scroll_up","mousedown .galaxy-frame-scroll-down":"event_panel_scroll_down","mousedown .f-close":"event_frame_close","mousedown .f-pin":"event_frame_lock"};this.delegateEvents(this.events)},event_frame_mouse_down:function(c){if(this.event.type!==null){return}if($(c.target).hasClass("f-header")||$(c.target).hasClass("f-title")){this.event.type="drag"}if($(c.target).hasClass("f-resize")){this.event.type="resize"}if(this.event.type===null){return}c.preventDefault();this.event.target=this.event_get_frame(c.target);if(this.event.target.grid_lock){this.event.type=null;return}this.event.xy={x:c.originalEvent.pageX,y:c.originalEvent.pageY};this.frame_drag_start(this.event.target)},event_frame_mouse_move:function(i){if(this.event.type!="drag"&&this.event.type!="resize"){return}var g={x:i.originalEvent.pageX,y:i.originalEvent.pageY};var d={x:g.x-this.event.xy.x,y:g.y-this.event.xy.y};this.event.xy=g;var h=this.frame_screen(this.event.target);if(this.event.type=="resize"){h.width+=d.x;h.height+=d.y;var f=this.options.cell-this.options.margin-1;h.width=Math.max(h.width,f);h.height=Math.max(h.height,f);this.frame_resize(this.event.target,h);h.width=this.to_grid_coord("width",h.width)+1;h.height=this.to_grid_coord("height",h.height)+1;h.width=this.to_pixel_coord("width",h.width);h.height=this.to_pixel_coord("height",h.height);this.frame_resize(this.galaxy_frame_shadow,h);this.frame_insert(this.galaxy_frame_shadow,{top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)})}if(this.event.type=="drag"){h.left+=d.x;h.top+=d.y;this.frame_offset(this.event.target,h);var c={top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)};if(c.left!==0){c.left++}this.frame_insert(this.galaxy_frame_shadow,c)}},event_frame_mouse_up:function(c){if(this.event.type!="drag"&&this.event.type!="resize"){return}this.frame_drag_stop(this.event.target);this.event.type=null},event_frame_close:function(d){if(this.event.type!==null){return}d.preventDefault();var f=this.event_get_frame(d.target);var c=this;$(f.id).fadeOut("fast",function(){$(f.id).remove();delete c.frame_list[f.id];c.frame_counter--;c.panel_refresh(true);c.panel_animation_complete();if(c.visible&&c.frame_counter==0){c.panel_show_hide()}})},event_frame_lock:function(c){if(this.event.type!==null){return}c.preventDefault();var d=this.event_get_frame(c.target);if(d.grid_lock){d.grid_lock=false;$(d.id).find(".f-pin").removeClass("f-toggle");$(d.id).find(".f-header").removeClass("f-not-allowed");$(d.id).find(".f-title").removeClass("f-not-allowed");$(d.id).find(".f-resize").show();$(d.id).find(".f-close").show()}else{d.grid_lock=true;$(d.id).find(".f-pin").addClass("f-toggle");$(d.id).find(".f-header").addClass("f-not-allowed");$(d.id).find(".f-title").addClass("f-not-allowed");$(d.id).find(".f-resize").hide();$(d.id).find(".f-close").hide()}},event_panel_load:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_show_hide()},event_panel_active:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_active_disable()},event_panel_scroll:function(c){if(this.event.type!==null||!this.visible){return}c.preventDefault();var d=c.originalEvent.detail?c.originalEvent.detail:c.originalEvent.wheelDelta/-3;this.panel_scroll(d)},event_panel_scroll_up:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_scroll(-this.options.scroll)},event_panel_scroll_down:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_scroll(this.options.scroll)},event_get_frame:function(c){return this.frame_list["#"+$(c).closest(".galaxy-frame").attr("id")]},frame_drag_start:function(d){this.frame_focus(d,true);var c=this.frame_screen(d);this.frame_resize(this.galaxy_frame_shadow,c);this.frame_grid(this.galaxy_frame_shadow,d.grid_location);d.grid_location=null;$(this.galaxy_frame_shadow.id).show();$(".f-cover").show()},frame_drag_stop:function(d){this.frame_focus(d,false);var c=this.frame_screen(this.galaxy_frame_shadow);this.frame_resize(d,c);this.frame_grid(d,this.galaxy_frame_shadow.grid_location,true);this.galaxy_frame_shadow.grid_location=null;$(this.galaxy_frame_shadow.id).hide();$(".f-cover").hide();this.panel_animation_complete()},to_grid_coord:function(e,d){var c=(e=="width"||e=="height")?1:-1;if(e=="top"){d-=this.top}return parseInt((d+c*this.options.margin)/this.options.cell,10)},to_pixel_coord:function(e,f){var c=(e=="width"||e=="height")?1:-1;var d=(f*this.options.cell)-c*this.options.margin;if(e=="top"){d+=this.top}return d},to_grid:function(c){return{top:this.to_grid_coord("top",c.top),left:this.to_grid_coord("left",c.left),width:this.to_grid_coord("width",c.width),height:this.to_grid_coord("height",c.height)}},to_pixel:function(c){return{top:this.to_pixel_coord("top",c.top),left:this.to_pixel_coord("left",c.left),width:this.to_pixel_coord("width",c.width),height:this.to_pixel_coord("height",c.height)}},is_collision:function(e){function c(h,g){return !(h.left>g.left+g.width-1||h.left+h.width-1g.top+g.height-1||h.top+h.height-1this.frame_counter){this.top=this.options.top_min;d.width=this.to_pixel_coord("width",this.options.frame.cols);d.height=this.to_pixel_coord("height",this.options.frame.rows);$(this.el).append(this.frame_template(e.substring(1),d.title,d.type,d.content));var f={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};this.frame_counter++;this.frame_list[e]=f;this.frame_resize(f,{width:d.width,height:d.height});this.frame_insert(f,{top:0,left:0},true);if(!this.visible){this.panel_show_hide()}}else{alert("You have reached the maximum number of allowed frames ("+this.options.frame_max+").")}},frame_insert:function(j,c,e){var d=[];if(j){j.grid_location=null;d.push([j,this.location_rank(c)])}var g=null;for(g in this.frame_list){var h=this.frame_list[g];if(h.grid_location!==null&&!h.grid_lock){h.grid_location=null;d.push([h,h.grid_rank])}}d.sort(function(k,f){var m=k[1];var l=f[1];return ml?1:0)});for(g=0;g'}return'
'+e+'
'+d+'
'},frame_template_shadow:function(c){return'
'},frame_template_background:function(){return'
'},frame_template_header:function(){return'
0
'},frame_template_menu:function(){return'
'}});return{GalaxyFrameManager:a}}); \ No newline at end of file diff --git a/static/scripts/packed/galaxy.pages.js b/static/scripts/packed/galaxy.pages.js new file mode 100644 index 00000000000..871ccedf68a --- /dev/null +++ b/static/scripts/packed/galaxy.pages.js @@ -0,0 +1 @@ +var Galaxy={ITEM_HISTORY:"item_history",ITEM_DATASET:"item_dataset",ITEM_WORKFLOW:"item_workflow",ITEM_PAGE:"item_page",ITEM_VISUALIZATION:"item_visualization",DIALOG_HISTORY_LINK:"link_history",DIALOG_DATASET_LINK:"link_dataset",DIALOG_WORKFLOW_LINK:"link_workflow",DIALOG_PAGE_LINK:"link_page",DIALOG_VISUALIZATION_LINK:"link_visualization",DIALOG_EMBED_HISTORY:"embed_history",DIALOG_EMBED_DATASET:"embed_dataset",DIALOG_EMBED_WORKFLOW:"embed_workflow",DIALOG_EMBED_PAGE:"embed_page",DIALOG_EMBED_VISUALIZATION:"embed_visualization",DIALOG_HISTORY_ANNOTATE:"history_annotate",};function init_galaxy_elts(a){$(".annotation",a._doc.body).each(function(){$(this).click(function(){var b=a._doc.createRange();b.selectNodeContents(this);var d=window.getSelection();d.removeAllRanges();d.addRange(b);var c=""})})}function get_item_info(d){var f,c,b;switch(d){case (Galaxy.ITEM_HISTORY):f="History";c="Histories";b="history";item_class="History";break;case (Galaxy.ITEM_DATASET):f="Dataset";c="Datasets";b="dataset";item_class="HistoryDatasetAssociation";break;case (Galaxy.ITEM_WORKFLOW):f="Workflow";c="Workflows";b="workflow";item_class="StoredWorkflow";break;case (Galaxy.ITEM_PAGE):f="Page";c="Pages";b="page";item_class="Page";break;case (Galaxy.ITEM_VISUALIZATION):f="Visualization";c="Visualizations";b="visualization";item_class="Visualization";break}var e="list_"+c.toLowerCase()+"_for_selection";var a=list_objects_url.replace("LIST_ACTION",e);return{singular:f,plural:c,controller:b,iclass:item_class,list_ajax_url:a}}function make_item_importable(a,c,b){ajax_url=set_accessible_url.replace("ITEM_CONTROLLER",a);$.ajax({type:"POST",url:ajax_url,data:{id:c,accessible:"True"},error:function(){alert("Making "+b+" accessible failed")}})}WYMeditor.editor.prototype.dialog=function(i,e,g){var a=this;var b=a.uniqueStamp();var f=a.selected();function h(){$("#set_link_id").click(function(){$("#link_attribute_label").text("ID/Name");var k=$(".wym_href");k.addClass("wym_id").removeClass("wym_href");if(f){k.val($(f).attr("id"))}$(this).remove()})}if(i==WYMeditor.DIALOG_LINK){if(f){$(a._options.hrefSelector).val($(f).attr(WYMeditor.HREF));$(a._options.srcSelector).val($(f).attr(WYMeditor.SRC));$(a._options.titleSelector).val($(f).attr(WYMeditor.TITLE));$(a._options.altSelector).val($(f).attr(WYMeditor.ALT))}var c,d;if(f){c=$(f).attr("href");if(c==undefined){c=""}d=$(f).attr("title");if(d==undefined){d=""}}show_modal("Create Link","

",{"Make link":function(){var l=$(a._options.hrefSelector).val(),m=$(".wym_title").val();if(l&&m){a._exec(WYMeditor.CREATE_LINK,b);var k=$("a[href="+b+"]",a._doc.body);k.attr(WYMeditor.HREF,l).attr(WYMeditor.TITLE,$(a._options.titleSelector).val()).attr("id",m);if(k.text().indexOf("wym-")===0){k.text(m)}}hide_modal()},Cancel:function(){hide_modal()}},{},h)}if(i==WYMeditor.DIALOG_IMAGE){if(a._selected_image){$(a._options.dialogImageSelector+" "+a._options.srcSelector).val($(a._selected_image).attr(WYMeditor.SRC));$(a._options.dialogImageSelector+" "+a._options.titleSelector).val($(a._selected_image).attr(WYMeditor.TITLE));$(a._options.dialogImageSelector+" "+a._options.altSelector).val($(a._selected_image).attr(WYMeditor.ALT))}show_modal("Image","



",{Insert:function(){var k=$(a._options.srcSelector).val();if(k.length>0){a._exec(WYMeditor.INSERT_IMAGE,b);$("img[src$="+b+"]",a._doc.body).attr(WYMeditor.SRC,k).attr(WYMeditor.TITLE,$(a._options.titleSelector).val()).attr(WYMeditor.ALT,$(a._options.altSelector).val())}hide_modal()},Cancel:function(){hide_modal()}});return}if(i==WYMeditor.DIALOG_TABLE){show_modal("Table","


",{Insert:function(){var o=$(a._options.rowsSelector).val();var r=$(a._options.colsSelector).val();if(o>0&&r>0){var n=a._doc.createElement(WYMeditor.TABLE);var l=null;var q=null;var k=$(a._options.captionSelector).val();var p=n.createCaption();p.innerHTML=k;for(x=0;xMake the selected "+j.plural.toLowerCase()+" accessible so that they can viewed by everyone.
",{Insert:function(){var m=false;if($("#make-importable:checked").val()!==null){m=true}var l=new Array();$("input[name=id]:checked").each(function(){var n=$(this).val();if(m){make_item_importable(j.controller,n,j.singular)}url_template=get_name_and_link_url+n;ajax_url=url_template.replace("ITEM_CONTROLLER",j.controller);$.getJSON(ajax_url,function(p){a._exec(WYMeditor.CREATE_LINK,b);var o=$("a[href="+b+"]",a._doc.body).text();if(o==""||o==b){a.insert(""+j.singular+" '"+p.name+"'")}else{$("a[href="+b+"]",a._doc.body).attr(WYMeditor.HREF,p.link).attr(WYMeditor.TITLE,j.singular+n)}})});hide_modal()},Cancel:function(){hide_modal()}})}})}if(i==Galaxy.DIALOG_EMBED_HISTORY||i==Galaxy.DIALOG_EMBED_DATASET||i==Galaxy.DIALOG_EMBED_WORKFLOW||i==Galaxy.DIALOG_EMBED_PAGE||i==Galaxy.DIALOG_EMBED_VISUALIZATION){var j;switch(i){case (Galaxy.DIALOG_EMBED_HISTORY):j=get_item_info(Galaxy.ITEM_HISTORY);break;case (Galaxy.DIALOG_EMBED_DATASET):j=get_item_info(Galaxy.ITEM_DATASET);break;case (Galaxy.DIALOG_EMBED_WORKFLOW):j=get_item_info(Galaxy.ITEM_WORKFLOW);break;case (Galaxy.DIALOG_EMBED_PAGE):j=get_item_info(Galaxy.ITEM_PAGE);break;case (Galaxy.DIALOG_EMBED_VISUALIZATION):j=get_item_info(Galaxy.ITEM_VISUALIZATION);break}$.ajax({url:j.list_ajax_url,data:{},error:function(){alert("Failed to list "+j.plural.toLowerCase()+" for selection")},success:function(k){if(i==Galaxy.DIALOG_EMBED_HISTORY||i==Galaxy.DIALOG_EMBED_WORKFLOW||i==Galaxy.DIALOG_EMBED_VISUALIZATION){k=k+"
Make the selected "+j.plural.toLowerCase()+" accessible so that they can viewed by everyone.
"}show_modal("Embed "+j.plural,k,{Embed:function(){var l=false;if($("#make-importable:checked").val()!=null){l=true}$("input[name=id]:checked").each(function(){var m=$(this).val();var p=$("label[for='"+m+"']:first").text();if(l){make_item_importable(j.controller,m,j.singular)}var n=j.iclass+"-"+m;var o="

Embedded Galaxy "+j.singular+" '"+p+"'

[Do not edit this block; Galaxy will fill it in with the annotated "+j.singular.toLowerCase()+" when it is displayed.]

";a.insert(" ");a.insert(o);$("#"+n,a._doc.body).each(function(){var q=true;while(q){var r=$(this).prev();if(r.length!=0&&jQuery.trim(r.text())==""){r.remove()}else{q=false}}})});hide_modal()},Cancel:function(){hide_modal()}})}})}if(i==Galaxy.DIALOG_ANNOTATE_HISTORY){$.ajax({url:list_histories_for_selection_url,data:{},error:function(){alert("Grid refresh failed")},success:function(k){show_modal("Insert Link to History",k,{Annotate:function(){var l=new Array();$("input[name=id]:checked").each(function(){var m=$(this).val();$.ajax({url:get_history_annotation_table_url,data:{id:m},error:function(){alert("Grid refresh failed")},success:function(n){a.insert(n);init_galaxy_elts(a)}})});hide_modal()},Cancel:function(){hide_modal()}})}})}};$(function(){$(document).ajaxError(function(i,g){var h=g.responseText||g.statusText||"Could not connect to server";show_modal("Server error",h,{"Ignore error":hide_modal});return false});$("[name=page_content]").wymeditor({skin:"galaxy",basePath:editor_base_path,iframeBasePath:iframe_base_path,boxHtml:"
"+WYMeditor.TOOLS+"
"+WYMeditor.IFRAME+WYMeditor.STATUS+"
",toolsItems:[{name:"Bold",title:"Strong",css:"wym_tools_strong"},{name:"Italic",title:"Emphasis",css:"wym_tools_emphasis"},{name:"Superscript",title:"Superscript",css:"wym_tools_superscript"},{name:"Subscript",title:"Subscript",css:"wym_tools_subscript"},{name:"InsertOrderedList",title:"Ordered_List",css:"wym_tools_ordered_list"},{name:"InsertUnorderedList",title:"Unordered_List",css:"wym_tools_unordered_list"},{name:"Indent",title:"Indent",css:"wym_tools_indent"},{name:"Outdent",title:"Outdent",css:"wym_tools_outdent"},{name:"Undo",title:"Undo",css:"wym_tools_undo"},{name:"Redo",title:"Redo",css:"wym_tools_redo"},{name:"CreateLink",title:"Link",css:"wym_tools_link"},{name:"Unlink",title:"Unlink",css:"wym_tools_unlink"},{name:"InsertImage",title:"Image",css:"wym_tools_image"},{name:"InsertTable",title:"Table",css:"wym_tools_table"},]});var d=$.wymeditors(0);var f=function(g){show_modal("Saving page","progress");$.ajax({url:save_url,type:"POST",data:{id:page_id,content:d.xhtml(),annotations:JSON.stringify(new Object()),_:"true"},success:function(){g()}})};$("#save-button").click(function(){f(function(){hide_modal()})});$("#close-button").click(function(){var h=false;if(h){var g=function(){window.onbeforeunload=undefined;window.document.location=page_list_url};show_modal("Close editor","There are unsaved changes to your page which will be lost.",{Cancel:hide_modal,"Save Changes":function(){f(g)}},{"Don't Save":g})}else{window.document.location=page_list_url}});var a=$("
Paragraph type
");$(".wym_area_top").append(a);var b={};$.each(d._options.containersItems,function(h,g){var i=g.name;b[g.title.replace("_"," ")]=function(){d.container(i)}});make_popupmenu(a,b);var c=$("").addClass("galaxy-page-editor-button");$(".wym_area_top").append(c);make_popupmenu(c,{"Insert History Link":function(){d.dialog(Galaxy.DIALOG_HISTORY_LINK)},"Insert Dataset Link":function(){d.dialog(Galaxy.DIALOG_DATASET_LINK)},"Insert Workflow Link":function(){d.dialog(Galaxy.DIALOG_WORKFLOW_LINK)},"Insert Page Link":function(){d.dialog(Galaxy.DIALOG_PAGE_LINK)},"Insert Visualization Link":function(){d.dialog(Galaxy.DIALOG_VISUALIZATION_LINK)},});var e=$("").addClass("galaxy-page-editor-button");$(".wym_area_top").append(e);make_popupmenu(e,{"Embed History":function(){d.dialog(Galaxy.DIALOG_EMBED_HISTORY)},"Embed Dataset":function(){d.dialog(Galaxy.DIALOG_EMBED_DATASET)},"Embed Workflow":function(){d.dialog(Galaxy.DIALOG_EMBED_WORKFLOW)},"Embed Visualization":function(){d.dialog(Galaxy.DIALOG_EMBED_VISUALIZATION)},})}); \ No newline at end of file diff --git a/static/scripts/packed/libs/jquery/jquery.wymeditor.js b/static/scripts/packed/libs/jquery/jquery.wymeditor.js index ae68ff07316..eafb2244ac1 100644 --- a/static/scripts/packed/libs/jquery/jquery.wymeditor.js +++ b/static/scripts/packed/libs/jquery/jquery.wymeditor.js @@ -1 +1 @@ -if(!WYMeditor){var WYMeditor={}}(function(){if(!window.console||!console.firebug){var b=["log","debug","info","warn","error","assert","dir","dirxml","group","groupEnd","time","timeEnd","count","trace","profile","profileEnd"];WYMeditor.console={};for(var a=0;a
"+WYMeditor.TOOLS+"
"+WYMeditor.CONTAINERS+WYMeditor.CLASSES+"
"+WYMeditor.HTML+WYMeditor.IFRAME+WYMeditor.STATUS+"
"+WYMeditor.LOGO+"
",logoHtml:"WYMeditor",iframeHtml:"
",editorStyles:[],toolsHtml:"

{Tools}

    "+WYMeditor.TOOLS_ITEMS+"
",toolsItemHtml:"
  • "+WYMeditor.TOOL_TITLE+"
    • ",toolsItems:[{name:"Bold",title:"Strong",css:"wym_tools_strong"},{name:"Italic",title:"Emphasis",css:"wym_tools_emphasis"},{name:"Superscript",title:"Superscript",css:"wym_tools_superscript"},{name:"Subscript",title:"Subscript",css:"wym_tools_subscript"},{name:"InsertOrderedList",title:"Ordered_List",css:"wym_tools_ordered_list"},{name:"InsertUnorderedList",title:"Unordered_List",css:"wym_tools_unordered_list"},{name:"Indent",title:"Indent",css:"wym_tools_indent"},{name:"Outdent",title:"Outdent",css:"wym_tools_outdent"},{name:"Undo",title:"Undo",css:"wym_tools_undo"},{name:"Redo",title:"Redo",css:"wym_tools_redo"},{name:"CreateLink",title:"Link",css:"wym_tools_link"},{name:"Unlink",title:"Unlink",css:"wym_tools_unlink"},{name:"InsertImage",title:"Image",css:"wym_tools_image"},{name:"InsertTable",title:"Table",css:"wym_tools_table"},{name:"Paste",title:"Paste_From_Word",css:"wym_tools_paste"},{name:"ToggleHtml",title:"HTML",css:"wym_tools_html"},{name:"Preview",title:"Preview",css:"wym_tools_preview"}],containersHtml:"

    {Containers}

      "+WYMeditor.CONTAINERS_ITEMS+"
    ",containersItemHtml:"
  • "+WYMeditor.CONTAINER_TITLE+"
    • ",containersItems:[{name:"P",title:"Paragraph",css:"wym_containers_p"},{name:"H1",title:"Heading_1",css:"wym_containers_h1"},{name:"H2",title:"Heading_2",css:"wym_containers_h2"},{name:"H3",title:"Heading_3",css:"wym_containers_h3"},{name:"H4",title:"Heading_4",css:"wym_containers_h4"},{name:"H5",title:"Heading_5",css:"wym_containers_h5"},{name:"H6",title:"Heading_6",css:"wym_containers_h6"},{name:"PRE",title:"Preformatted",css:"wym_containers_pre"},{name:"BLOCKQUOTE",title:"Blockquote",css:"wym_containers_blockquote"},{name:"TH",title:"Table_Header",css:"wym_containers_th"}],classesHtml:"

    {Classes}

      "+WYMeditor.CLASSES_ITEMS+"
    ",classesItemHtml:"
  • "+WYMeditor.CLASS_TITLE+"
    • ",classesItems:[],statusHtml:"

    {Status}

    ",htmlHtml:"

    {Source_Code}

    ",boxSelector:".wym_box",toolsSelector:".wym_tools",toolsListSelector:" ul",containersSelector:".wym_containers",classesSelector:".wym_classes",htmlSelector:".wym_html",iframeSelector:".wym_iframe iframe",iframeBodySelector:".wym_iframe",statusSelector:".wym_status",toolSelector:".wym_tools a",containerSelector:".wym_containers a",classSelector:".wym_classes a",htmlValSelector:".wym_html_val",hrefSelector:".wym_href",srcSelector:".wym_src",titleSelector:".wym_title",altSelector:".wym_alt",textSelector:".wym_text",rowsSelector:".wym_rows",colsSelector:".wym_cols",captionSelector:".wym_caption",summarySelector:".wym_summary",submitSelector:".wym_submit",cancelSelector:".wym_cancel",previewSelector:"",dialogTypeSelector:".wym_dialog_type",dialogLinkSelector:".wym_dialog_link",dialogImageSelector:".wym_dialog_image",dialogTableSelector:".wym_dialog_table",dialogPasteSelector:".wym_dialog_paste",dialogPreviewSelector:".wym_dialog_preview",updateSelector:".wymupdate",updateEvent:"click",dialogFeatures:"menubar=no,titlebar=no,toolbar=no,resizable=no,width=560,height=300,top=0,left=0",dialogFeaturesPreview:"menubar=no,titlebar=no,toolbar=no,resizable=no,scrollbars=yes,width=560,height=300,top=0,left=0",dialogHtml:""+WYMeditor.DIALOG_TITLE+" - - - <%def name="javascripts()"> ${parent.javascripts()} - ${h.js( "libs/jquery/jquery.event.drag", "libs/jquery/jquery.event.drop", "libs/jquery/jquery.event.hover", "libs/jquery/jquery.form", - "libs/jquery/jstorage", "libs/jquery/jquery.wymeditor", "libs/jquery/jquery.autocomplete", "galaxy.autocom_tagging")} - - + ${h.js( "libs/jquery/jquery.form", "libs/jquery/jstorage", "libs/jquery/jquery.wymeditor", "libs/jquery/jquery.autocomplete", "galaxy.autocom_tagging", "galaxy.pages")} <%def name="stylesheets()"> @@ -782,7 +53,7 @@
    - +
    From 9bb3ca8ba1fd5aaaa2d032a8cbd7723310d831bc Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 2 Jul 2013 04:12:45 -0400 Subject: [PATCH 242/318] Bugfix for distributed object store free space monitor thread. --- lib/galaxy/objectstore/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 01abe2d6c6a..617feccad93 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -880,7 +880,8 @@ class DistributedObjectStore(ObjectStore): self.__parse_distributed_config(config) - if self.global_max_percent_full or filter(lambda x: x is not None, self.max_percent_full.values()): + self.sleeper = None + if self.global_max_percent_full or filter(lambda x: x != 0.0, self.max_percent_full.values()): self.sleeper = Sleeper() self.filesystem_monitor_thread = threading.Thread(target=self.__filesystem_monitor) self.filesystem_monitor_thread.start() @@ -931,7 +932,8 @@ class DistributedObjectStore(ObjectStore): def shutdown(self): super(DistributedObjectStore, self).shutdown() - self.sleeper.wake() + if self.sleeper is not None: + self.sleeper.wake() def exists(self, obj, **kwargs): return self.__call_method('exists', obj, False, False, **kwargs) From 828c4953f524459cd130532515534adf2721a90f Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 2 Jul 2013 04:13:02 -0400 Subject: [PATCH 243/318] Upgrade pbs_python to 4.3.5 --- eggs.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eggs.ini b/eggs.ini index 872ee3c0caa..26e0907b9ae 100644 --- a/eggs.ini +++ b/eggs.ini @@ -19,7 +19,7 @@ MarkupSafe = 0.12 mercurial = 2.2.3 MySQL_python = 1.2.3c1 numpy = 1.6.0 -pbs_python = 4.1.0 +pbs_python = 4.3.5 psycopg2 = 2.0.13 pycrypto = 2.5 pysam = 0.4.2 From 8a461dfac2becd0335edd2b820f4cd5755824ae0 Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Tue, 2 Jul 2013 09:25:55 -0400 Subject: [PATCH 244/318] Make SAM_to_BAM tool write temporary files to job working directory rather than generic temp space. --- tools/samtools/sam_to_bam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/samtools/sam_to_bam.py b/tools/samtools/sam_to_bam.py index f209a7e41c0..4a6b8a19bda 100644 --- a/tools/samtools/sam_to_bam.py +++ b/tools/samtools/sam_to_bam.py @@ -42,7 +42,7 @@ def __main__(): except: sys.stdout.write( 'Could not determine Samtools version\n' ) - tmp_dir = tempfile.mkdtemp() + tmp_dir = tempfile.mkdtemp( dir='.' ) if not options.ref_file or options.ref_file == 'None': # We're using locally cached reference sequences( e.g., /galaxy/data/equCab2/sam_index/equCab2.fa ). # The indexes for /galaxy/data/equCab2/sam_index/equCab2.fa will be contained in From 46eee3acda098258f2da876a278406526cc809e5 Mon Sep 17 00:00:00 2001 From: Brad Chapman Date: Mon, 1 Jul 2013 20:24:07 -0400 Subject: [PATCH 245/318] Add new engine parameter to downgrade function for 0112 --- ...story_association_and_data_manager_job_association_tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/migrate/versions/0112_add_data_manager_history_association_and_data_manager_job_association_tables.py b/lib/galaxy/model/migrate/versions/0112_add_data_manager_history_association_and_data_manager_job_association_tables.py index b3c60225688..e6806a2e9c3 100644 --- a/lib/galaxy/model/migrate/versions/0112_add_data_manager_history_association_and_data_manager_job_association_tables.py +++ b/lib/galaxy/model/migrate/versions/0112_add_data_manager_history_association_and_data_manager_job_association_tables.py @@ -54,7 +54,7 @@ def upgrade(migrate_engine): log.debug( "Creating data_manager_job_association table failed: %s" % str( e ) ) -def downgrade(): +def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() try: From b35a6c79926cbbca5455d7b2b338488041cbbdba Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Tue, 2 Jul 2013 10:48:31 -0400 Subject: [PATCH 246/318] Fix two more downgrade invocations to accept the migrate_engine parameter --- .../model/migrate/versions/0005_drop_tool_related_tables.py | 2 +- .../versions/0018_add_repository_metadata_flag_columns.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/model/migrate/versions/0005_drop_tool_related_tables.py b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0005_drop_tool_related_tables.py index 152653997aa..2d7e233b223 100644 --- a/lib/galaxy/webapps/tool_shed/model/migrate/versions/0005_drop_tool_related_tables.py +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0005_drop_tool_related_tables.py @@ -94,7 +94,7 @@ def upgrade(migrate_engine): Tool_table.drop() except Exception, e: log.debug( "Dropping tool table failed: %s" % str( e ) ) -def downgrade(): +def downgrade(migrate_engine): # Load existing tables metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/webapps/tool_shed/model/migrate/versions/0018_add_repository_metadata_flag_columns.py b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0018_add_repository_metadata_flag_columns.py index 7085c87fab0..42ce9f2ddc0 100644 --- a/lib/galaxy/webapps/tool_shed/model/migrate/versions/0018_add_repository_metadata_flag_columns.py +++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0018_add_repository_metadata_flag_columns.py @@ -64,7 +64,7 @@ def upgrade(migrate_engine): except Exception, e: print "Adding missing_test_components column to the repository_metadata table failed: %s" % str( e ) -def downgrade(): +def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() # Drop missing_test_components and tool_test_results from the repository_metadata table and add tool_test_errors to the repository_metadata table. From 0b6f0708c8932ed313cc9caa6bd316543ca94eb0 Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Wed, 3 Jul 2013 10:39:02 -0400 Subject: [PATCH 247/318] Make BAM_to_SAM tool write temporary files to job working directory rather than generic temp space. --- tools/samtools/bam_to_sam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/samtools/bam_to_sam.py b/tools/samtools/bam_to_sam.py index 310d4b70a6e..17fb8688ef7 100644 --- a/tools/samtools/bam_to_sam.py +++ b/tools/samtools/bam_to_sam.py @@ -43,7 +43,7 @@ def __main__(): except: sys.stdout.write( 'Could not determine Samtools version\n' ) - tmp_dir = tempfile.mkdtemp() + tmp_dir = tempfile.mkdtemp( dir='.' ) try: # exit if input file empty From 2ae17f816daa4976b3635796bf1b8f5d0d17e7fd Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Sat, 6 Jul 2013 19:41:10 -0400 Subject: [PATCH 248/318] Allow public usernames in the tool shed to be a minimum of 3 characters. --- lib/galaxy/security/validate_user_input.py | 8 ++++++-- tools/data_source/upload.xml | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/security/validate_user_input.py b/lib/galaxy/security/validate_user_input.py index 0d3b68921ac..705519df9ff 100644 --- a/lib/galaxy/security/validate_user_input.py +++ b/lib/galaxy/security/validate_user_input.py @@ -23,8 +23,12 @@ def validate_publicname( trans, publicname, user=None ): return '' if user and user.username == publicname: return '' - if len( publicname ) < 4: - return "Public name must be at least 4 characters in length" + if trans.webapp.name == 'tool_shed': + if len( publicname ) < 3: + return "Public name must be at least 3 characters in length" + else: + if len( publicname ) < 4: + return "Public name must be at least 4 characters in length" if len( publicname ) > 255: return "Public name cannot be more than 255 characters in length" if not( VALID_PUBLICNAME_RE.match( publicname ) ): diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index de02c4afd36..e15de2ffbd2 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -33,7 +33,7 @@ - + not ( ( isinstance( value, unicode ) or isinstance( value, str ) ) and value != "" ) From 86d0d802af575a3e576407223fc74c96ecf98a28 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Mon, 8 Jul 2013 11:31:29 -0400 Subject: [PATCH 249/318] Contributions from Bjorn Gruning - add support for a template_command action type in tool dependency definitions for the tool shed. An example of the new action tag is .... This is a slightly modified version of Bjorn's original patch contribution. --- lib/galaxy/util/template.py | 2 +- .../tool_dependencies/common_util.py | 16 +++++++--- .../tool_dependencies/fabric_util.py | 21 ++++++++++++-- .../tool_dependencies/install_util.py | 29 ++++++++++++++----- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/util/template.py b/lib/galaxy/util/template.py index a721c969bfd..ac89f91668a 100644 --- a/lib/galaxy/util/template.py +++ b/lib/galaxy/util/template.py @@ -6,4 +6,4 @@ from Cheetah.Template import Template def fill_template( template_text, context=None, **kwargs ): if not context: context = kwargs - return str( Template( source=template_text, searchList=[context] ) ) \ No newline at end of file + return str( Template( source=template_text, searchList=[context] ) ) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py index 45b11ed3949..9c48ee5eeb0 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/common_util.py @@ -80,10 +80,6 @@ def extract_tar( file_name, file_path ): tar.extractall( path=file_path ) tar.close() -def format_traceback(): - ex_type, ex, tb = sys.exc_info() - return ''.join( traceback.format_tb( tb ) ) - def extract_zip( archive_path, extraction_path ): # TODO: change this method to use zipfile.Zipfile.extractall() when we stop supporting Python 2.5. if not zipfile_ok( archive_path ): @@ -99,6 +95,10 @@ def extract_zip( archive_path, extraction_path ): zip_archive.close() return True +def format_traceback(): + ex_type, ex, tb = sys.exc_info() + return ''.join( traceback.format_tb( tb ) ) + def get_env_shell_file_path( installation_directory ): env_shell_file_name = 'env.sh' default_location = os.path.abspath( os.path.join( installation_directory, env_shell_file_name ) ) @@ -167,6 +167,14 @@ def get_env_shell_file_paths( app, elem ): log.debug( error_message ) return env_shell_file_paths +def get_env_var_values( install_dir ): + env_var_dict = {} + env_var_dict[ 'INSTALL_DIR' ] = install_dir + env_var_dict[ 'system_install' ] = install_dir + # If the Python interpreter is 64bit then we can safely assume that the underlying system is also 64bit. + env_var_dict[ '__is64bit__' ] = sys.maxsize > 2**32 + return env_var_dict + def isbz2( file_path ): return checkers.is_bz2( file_path ) diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py index f5c56b1dfd1..19bd4378bd4 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py @@ -8,8 +8,9 @@ import shutil import tempfile import shutil from contextlib import contextmanager - +from galaxy.util.template import fill_template from galaxy import eggs + import pkg_resources pkg_resources.require('ssh' ) @@ -297,12 +298,28 @@ def install_and_build_package( app, tool_dependency, actions_dict ): with settings( warn_only=True ): cmd = '' for env_shell_file_path in env_shell_file_paths: - for i, env_setting in enumerate( open( env_shell_file_path ) ): + for env_setting in open( env_shell_file_path ): cmd += '%s\n' % env_setting cmd += action_dict[ 'command' ] return_code = handle_command( app, tool_dependency, install_dir, cmd ) if return_code: return + elif action_type == 'template_command': + env_vars = dict() + for env_shell_file_path in env_shell_file_paths: + for env_setting in open( env_shell_file_path ): + env_string = env_setting.split( ';' )[ 0 ] + env_name, env_path = env_string.split( '=' ) + env_vars[ env_name ] = env_path + env_vars.update( common_util.get_env_var_values( install_dir ) ) + language = action_dict[ 'language' ] + with settings( warn_only=True, **env_vars ): + if language == 'cheetah': + # We need to import fabric.api.env so that we can access all collected environment variables. + cmd = fill_template( '#from fabric.api import env\n%s' % action_dict[ 'command' ], context=env_vars ) + return_code = handle_command( app, tool_dependency, install_dir, cmd ) + if return_code: + return elif action_type == 'download_file': # Download a single file to the current working directory. url = action_dict[ 'url' ] diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py index d91b05b5ad6..2707eb04af5 100644 --- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py @@ -361,12 +361,8 @@ def install_via_fabric( app, tool_dependency, actions_elem, install_dir, package sa_session = app.model.context.current def evaluate_template( text ): - """ Substitute variables defined in XML blocks obtained loaded from dependencies file. """ - # # Added for compatibility with CloudBioLinux. - # TODO: Add tool_version substitution for compat with CloudBioLinux. - substitutions = { "INSTALL_DIR" : install_dir, - "system_install" : install_dir } - return Template( text ).safe_substitute( substitutions ) + """ Substitute variables defined in XML blocks from dependencies file.""" + return Template( text ).safe_substitute( common_util.get_env_var_values( install_dir ) ) if not os.path.exists( install_dir ): os.makedirs( install_dir ) @@ -387,6 +383,25 @@ def install_via_fabric( app, tool_dependency, actions_elem, install_dir, package action_dict[ 'command' ] = action_elem_text else: continue + elif action_type == 'template_command': + # Default to Cheetah as it's the first template language supported. + language = action_elem.get( 'language', 'cheetah' ).lower() + if language == 'cheetah': + # Cheetah template syntax. + # + # #if env.PATH: + # make + # #end if + # + action_elem_text = action_elem.text.strip() + if action_elem_text: + action_dict[ 'language' ] = language + action_dict[ 'command' ] = action_elem_text + else: + continue + else: + log.debug( "Unsupported template language '%s'. Not proceeding." % str( language ) ) + raise Exception( "Unsupported template language '%s' in tool dependency definition." % str( language ) ) elif action_type == 'download_by_url': # http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2 if action_elem.text: @@ -690,4 +705,4 @@ def url_join( *args ): parts = [] for arg in args: parts.append( arg.strip( '/' ) ) - return '/'.join( parts ) \ No newline at end of file + return '/'.join( parts ) From 853fdc7b208e663f7bfede2604f02d06bfb8abf1 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Mon, 8 Jul 2013 12:03:22 -0400 Subject: [PATCH 250/318] Fix for checking the Galaxy environment for missing migrated tools at server startup. --- lib/tool_shed/util/common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index 3874740d319..b7653b2b2da 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -19,6 +19,7 @@ def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_scri tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ) # The default behavior is that the tool shed is down. tool_shed_accessible = False + missing_tool_configs_dict = odict() if tool_shed_url: for elem in root: if elem.tag == 'repository': @@ -48,7 +49,6 @@ def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_scri migrated_tool_configs_dict[ tool_elem.get( 'file' ) ] = tool_dependencies if tool_shed_accessible: # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. - missing_tool_configs_dict = odict() for tool_panel_config in tool_panel_configs: tree, error_message = xml_util.parse_xml( tool_panel_config ) if tree: From 4ce30a512349ee30630bc97a367f1a4491da1dc8 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Mon, 8 Jul 2013 14:31:15 -0400 Subject: [PATCH 251/318] Raise an exception with a useful message if a dependency definition file being uploaded to a repository in the tool shed is missing a required name or owner attribute for a repository dependency. --- lib/tool_shed/util/commit_util.py | 39 ++++++++++++++++--- lib/tool_shed/util/common_util.py | 2 +- .../util/repository_dependency_util.py | 2 +- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/lib/tool_shed/util/commit_util.py b/lib/tool_shed/util/commit_util.py index 1a9546ba4df..652ffbbdb4a 100644 --- a/lib/tool_shed/util/commit_util.py +++ b/lib/tool_shed/util/commit_util.py @@ -164,6 +164,18 @@ def handle_directory_changes( trans, repository, full_path, filenames_in_archive suc.handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) return True, '', files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed +def handle_missing_repository_attribute( elem ): + # + error_message = '' + name = elem.get( 'name' ) + if not name: + error_message += 'The tag is missing the required name attribute. ' + owner = elem.get( 'owner' ) + if not owner: + error_message += 'The tag is missing the required owner attribute. ' + log.debug( error_message ) + return error_message + def handle_gzip( repository, uploaded_file_name ): fd, uncompressed = tempfile.mkstemp( prefix='repo_%d_upload_gunzip_' % repository.id, dir=os.path.dirname( uploaded_file_name ), text=False ) gzipped_file = gzip.GzipFile( uploaded_file_name, 'rb' ) @@ -192,7 +204,10 @@ def handle_repository_dependencies_definition( trans, repository_dependencies_co if root.tag == 'repositories': for index, elem in enumerate( root ): # - populated, elem = handle_repository_dependency_elem( trans, elem ) + populated, elem, error_message = handle_repository_dependency_elem( trans, elem ) + if error_message: + exception_message = 'The repository_dependencies.xml file contains an invalid tag. %s' % error_message + raise Exception( exception_message ) if populated: root[ index ] = elem if not altered: @@ -202,6 +217,12 @@ def handle_repository_dependencies_definition( trans, repository_dependencies_co def handle_repository_dependency_elem( trans, elem ): # + error_message = '' + name = elem.get( 'name' ) + owner = elem.get( 'owner' ) + if not name or not owner: + error_message = handle_missing_repository_attribute( elem ) + return False, elem, error_message populated = False toolshed = elem.get( 'toolshed' ) if not toolshed: @@ -209,8 +230,6 @@ def handle_repository_dependency_elem( trans, elem ): toolshed = str( url_for( '/', qualified=True ) ).rstrip( '/' ) elem.attrib[ 'toolshed' ] = toolshed populated = True - name = elem.get( 'name' ) - owner = elem.get( 'owner' ) changeset_revision = elem.get( 'changeset_revision' ) if not changeset_revision: # Populate the changeset_revision attribute with the latest installable metadata revision for the defined repository. @@ -224,7 +243,9 @@ def handle_repository_dependency_elem( trans, elem ): if lastest_installable_changeset_revision != suc.INITIAL_CHANGELOG_HASH: elem.attrib[ 'changeset_revision' ] = lastest_installable_changeset_revision populated = True - return populated, elem + else: + error_message = 'Unable to locate repository with name %s and owner %s. ' % ( str( name ), str( owner ) ) + return populated, elem, error_message def handle_tool_dependencies_definition( trans, tool_dependencies_config ): altered = False @@ -241,7 +262,10 @@ def handle_tool_dependencies_definition( trans, tool_dependencies_config ): for package_index, package_elem in enumerate( root_elem ): if package_elem.tag == 'repository': # - populated, repository_elem = handle_repository_dependency_elem( trans, package_elem ) + populated, repository_elem, error_message = handle_repository_dependency_elem( trans, package_elem ) + if error_message: + exception_message = 'The tool_dependencies.xml file contains an invalid tag. %s' % error_message + raise Exception( exception_message ) if populated: root_elem[ package_index ] = repository_elem package_altered = True @@ -259,7 +283,10 @@ def handle_tool_dependencies_definition( trans, tool_dependencies_config ): # # for repo_index, repo_elem in enumerate( action_elem ): - populated, repository_elem = handle_repository_dependency_elem( trans, repo_elem ) + populated, repository_elem, error_message = handle_repository_dependency_elem( trans, repo_elem ) + if error_message: + exception_message = 'The tool_dependencies.xml file contains an invalid tag. %s' % error_message + raise Exception( exception_message ) if populated: action_elem[ repo_index ] = repository_elem package_altered = True diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index b7653b2b2da..7fc6c637a08 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -63,7 +63,7 @@ def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_scri else: exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % ( tool_shed, app.config.tool_sheds_config ) exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' - raise Exception( exception_msg ) + raise Exception( exception_msg ) return tool_shed_accessible, missing_tool_configs_dict def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ): diff --git a/lib/tool_shed/util/repository_dependency_util.py b/lib/tool_shed/util/repository_dependency_util.py index e2f561930cf..e80e993da5a 100644 --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -249,7 +249,7 @@ def generate_message_for_invalid_repository_dependencies( metadata_dict ): if metadata_dict: invalid_repository_dependencies_dict = metadata_dict.get( 'invalid_repository_dependencies', None ) if invalid_repository_dependencies_dict: - invalid_repository_dependencies = invalid_repository_dependencies_dict[ 'invalid_repository_dependencies' ] + invalid_repository_dependencies = invalid_repository_dependencies_dict.get( 'invalid_repository_dependencies', [] ) for repository_dependency_tup in invalid_repository_dependencies: toolshed, name, owner, changeset_revision, prior_installation_required, error = \ suc.parse_repository_dependency_tuple( repository_dependency_tup, contains_error=True ) From 026cdda755ea1de536aa171d81575c04cb66a01b Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Tue, 9 Jul 2013 11:54:49 -0400 Subject: [PATCH 252/318] Fixes for repairing an installed tool shed repository. --- .../galaxy/controllers/admin_toolshed.py | 32 ++++----- .../galaxy_install/repository_util.py | 68 +++++++++++++------ lib/tool_shed/util/tool_util.py | 5 +- .../repair_repository.mako | 2 +- 4 files changed, 64 insertions(+), 43 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index b3212b55d03..01127e81126 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -589,14 +589,6 @@ class AdminToolshed( AdminGalaxy ): else: kwd[ 'message' ] = 'All selected tool shed repositories are already installed.' kwd[ 'status' ] = 'error' - elif operation == "repair": - # In this case, tsridslist is ordered. - repositories_for_repair = [] - for tsr_id in tsridslist: - repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) - repositories_for_repair.append( repository ) - ordered_repo_info_dicts = kwd.get( 'ordered_repo_info_dicts', [] ) - self.repair_tool_shed_repositories( trans, repositories_for_repair, ordered_repo_info_dicts ) return self.repository_installation_grid( trans, **kwd ) @web.expose @@ -1159,23 +1151,29 @@ class AdminToolshed( AdminGalaxy ): status=status ) ) tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) if kwd.get( 'repair_repository_button', False ): - repair_dict = kwd.get( 'repair_dict', None ) + encoded_repair_dict = kwd.get( 'repair_dict', None ) + if encoded_repair_dict: + repair_dict = encoding_util.tool_shed_decode( encoded_repair_dict ) + else: + repair_dict = None if not repair_dict: repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository ) - ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) - ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) - if ordered_tsr_ids and ordered_repo_info_dicts: - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='manage_repositories', - operation='repair', - ordered_tsr_ids=ordered_tsr_ids, - ordered_repo_info_dicts=ordered_repo_info_dicts ) ) + ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) + ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) + if ordered_tsr_ids and ordered_repo_info_dicts: + repositories_for_repair = [] + for tsr_id in ordered_tsr_ids: + repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( tsr_id ) ) + repositories_for_repair.append( repository ) + return self.repair_tool_shed_repositories( trans, repositories_for_repair, ordered_repo_info_dicts ) tool_shed_repository = suc.get_installed_tool_shed_repository( trans, repository_id ) repair_dict = repository_util.get_repair_dict( trans, tool_shed_repository ) + encoded_repair_dict = encoding_util.tool_shed_encode( repair_dict ) ordered_tsr_ids = repair_dict.get( 'ordered_tsr_ids', [] ) ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) return trans.fill_template( 'admin/tool_shed_repository/repair_repository.mako', repository=tool_shed_repository, + encoded_repair_dict=encoded_repair_dict, repair_dict=repair_dict, message=message, status=status ) diff --git a/lib/tool_shed/galaxy_install/repository_util.py b/lib/tool_shed/galaxy_install/repository_util.py index 302c10ed89b..a61e5b638f0 100644 --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -110,9 +110,9 @@ def get_installed_repositories_from_repository_dependencies( trans, repository_d installed_repositories.append( installed_repository ) for rd_val in rd_vals: try: - tool_shed, name, owner, changeset_revision, prior_installation_required = container_util.get_components_from_key( rd_key ) + tool_shed, name, owner, changeset_revision, prior_installation_required = rd_val except: - tool_shed, name, owner, changeset_revision = container_util.get_components_from_key( rd_val ) + tool_shed, name, owner, changeset_revision = rd_val installed_repository = suc.get_tool_shed_repository_by_shed_name_owner_changeset_revision( trans.app, tool_shed, name, owner, changeset_revision ) if installed_repository not in installed_repositories: installed_repositories.append( installed_repository ) @@ -182,6 +182,7 @@ def get_repair_dict( trans, repository ): # tsr_ids to ensure all repositories are repaired in the required order. tsr_ids = [] repo_info_dicts = [] + tool_panel_section_keys = [] for installed_repository in installed_repositories: tsr_ids.append( trans.security.encode_id( installed_repository.id ) ) repository_clone_url = suc.generate_clone_url_for_installed_repository( trans.app, installed_repository ) @@ -189,8 +190,36 @@ def get_repair_dict( trans, repository ): metadata = installed_repository.metadata if metadata: tool_dependencies = metadata.get( 'tool_dependencies', None ) + tool_panel_section_dict = metadata.get( 'tool_panel_section', None ) + if tool_panel_section_dict: + # The installed_repository must be in the uninstalled state. The structure of tool_panel_section_dict is: + # { : [{ 'id':
    , 'name':
    , 'version':
    , 'tool_config': }]} + # Here is an example: + # {"localhost:9009/repos/test/filter/Filter1/1.1.0": + # [{"id": "filter_and_sort", "name": "Filter and Sort", "tool_config": "filtering.xml", "version": ""}]} + # Currently all tools contained within an installed tool shed repository must be loaded into the same section in the tool panel, so we can + # get the section id of the first guid in the tool_panel_section_dict. In the future, we'll have to handle different sections per guid. + guid = tool_panel_section_dict.keys()[ 0 ] + section_dicts = tool_panel_section_dict[ guid ] + section_dict = section_dicts[ 0 ] + tool_panel_section_id = section_dict[ 'id' ] + tool_panel_section_name = section_dict[ 'name' ] + if tool_panel_section_id: + tool_panel_section_key, tool_panel_section = tool_util.get_or_create_tool_section( trans, + tool_panel_section_id=tool_panel_section_id, + new_tool_panel_section=tool_panel_section_name ) + tool_panel_section_keys.append( tool_panel_section_key ) + else: + # The tools will be loaded outside of any sections in the tool panel. + tool_panel_section_keys.append( None ) + else: + # The installed_repository must be in the installed state, so we can skip determining if it has tools that are displayed in a tool panel section + # since no changes will be made to it. + tool_panel_section_keys.append( None ) else: tool_dependencies = None + # The tools will be loaded outside of any sections in the tool panel. + tool_panel_section_keys.append( None ) repo_info_dict = create_repo_info_dict( trans=trans, repository_clone_url=repository_clone_url, changeset_revision=installed_repository.changeset_revision, @@ -202,11 +231,11 @@ def get_repair_dict( trans, repository ): tool_dependencies=tool_dependencies, repository_dependencies=repository_dependencies ) repo_info_dicts.append( repo_info_dict ) - # We don't consider tool_panel_section_keys since the repository database records already exist. - ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = order_components_for_installation( trans, - tsr_ids, - repo_info_dicts, - tool_panel_section_keys=None ) + ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ + order_components_for_installation( trans, + tsr_ids, + repo_info_dicts, + tool_panel_section_keys=tool_panel_section_keys ) repair_dict[ 'ordered_tsr_ids' ] = ordered_tsr_ids repair_dict[ 'ordered_repo_info_dicts' ] = ordered_repo_info_dicts repair_dict[ 'ordered_tool_panel_section_keys' ] = ordered_tool_panel_section_keys @@ -245,7 +274,7 @@ def get_repo_info_dict( trans, repository_id, changeset_revision ): repository_dependencies=None ) return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, has_repository_dependencies -def get_repository_components_for_installation( encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys=None ): +def get_repository_components_for_installation( encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys ): """ The received encoded_tsr_ids, repo_info_dicts, and tool_panel_section_keys are 3 lists that contain associated elements at each location in the list. This method will return the elements from repo_info_dicts and tool_panel_section_keys associated with the received encoded_tsr_id @@ -254,10 +283,7 @@ def get_repository_components_for_installation( encoded_tsr_id, encoded_tsr_ids, for index, tsr_id in enumerate( encoded_tsr_ids ): if tsr_id == encoded_tsr_id: repo_info_dict = repo_info_dicts[ index ] - if tool_panel_section_keys: - tool_panel_section_key = tool_panel_section_keys[ index ] - else: - tool_panel_section_key = None + tool_panel_section_key = tool_panel_section_keys[ index ] return repo_info_dict, tool_panel_section_key return None, None @@ -684,7 +710,7 @@ def merge_containers_dicts_for_new_install( containers_dicts ): lock.release() return new_containers_dict -def order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys=None ): +def order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_panel_section_keys ): """ Some repositories may have repository dependencies that are required to be installed before the dependent repository. This method will inspect the list of repositories about to be installed and make sure to order them appropriately. For each repository about to be installed, if required repositories are not @@ -713,16 +739,14 @@ def order_components_for_installation( trans, tsr_ids, repo_info_dicts, tool_pan tool_panel_section_keys=tool_panel_section_keys ) ordered_tsr_ids.append( prior_install_required_id ) ordered_repo_info_dicts.append( prior_repo_info_dict ) - if tool_panel_section_keys: - ordered_tool_panel_section_keys.append( prior_tool_panel_section_key ) + ordered_tool_panel_section_keys.append( prior_tool_panel_section_key ) repo_info_dict, tool_panel_section_key = get_repository_components_for_installation( tsr_id, tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) ordered_tsr_ids.append( tsr_id ) ordered_repo_info_dicts.append( repo_info_dict ) - if tool_panel_section_keys: - ordered_tool_panel_section_keys.append( tool_panel_section_key ) + ordered_tool_panel_section_keys.append( tool_panel_section_key ) return ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys def populate_containers_dict_for_new_install( trans, tool_shed_url, tool_path, readme_files_dict, installed_repository_dependencies, missing_repository_dependencies, @@ -785,11 +809,11 @@ def repair_tool_shed_repository( trans, repository, repo_info_dict ): shed_tool_conf, tool_path, relative_install_dir = suc.get_tool_panel_config_tool_path_install_dir( trans.app, repository ) # Reset the repository attributes to the New state for installation. if metadata: - tool_panel_section_key = tool_util.handle_tool_panel_selection( trans, - metadata, - no_changes_checked=True, - tool_panel_section=None, - new_tool_panel_section=None ) + tool_section, new_tool_panel_section, tool_panel_section_key = tool_util.handle_tool_panel_selection( trans, + metadata, + no_changes_checked=True, + tool_panel_section=None, + new_tool_panel_section=None ) else: # The tools will be loaded outside of any sections in the tool panel. tool_panel_section_key = None diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index 8924a926395..c6578477935 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -639,8 +639,7 @@ def handle_tool_panel_selection( trans, metadata, no_changes_checked, tool_panel if 'tools' in metadata: # This forces everything to be loaded into the same section (or no section) in the tool panel. if no_changes_checked: - # Make sure the no_changes check box overrides the new_tool_panel_section if the user checked the check box and entered something - # into the field. + # Make sure the no_changes check box overrides the new_tool_panel_section if the user checked the check box and entered something into the field. new_tool_panel_section = None if 'tool_panel_section' in metadata: tool_panel_dict = metadata[ 'tool_panel_section' ] @@ -649,7 +648,7 @@ def handle_tool_panel_selection( trans, metadata, no_changes_checked, tool_panel else: tool_panel_dict = generate_tool_panel_dict_for_new_install( metadata[ 'tools' ] ) if tool_panel_dict: - #tool_panel_dict is empty when tools exist but are not installed into a tool panel + # The tool_panel_dict is empty when tools exist but are not installed into a tool panel section. tool_section_dicts = tool_panel_dict[ tool_panel_dict.keys()[ 0 ] ] tool_section_dict = tool_section_dicts[ 0 ] original_section_id = tool_section_dict[ 'id' ] diff --git a/templates/admin/tool_shed_repository/repair_repository.mako b/templates/admin/tool_shed_repository/repair_repository.mako index 721d4da829f..7351073325f 100644 --- a/templates/admin/tool_shed_repository/repair_repository.mako +++ b/templates/admin/tool_shed_repository/repair_repository.mako @@ -25,7 +25,7 @@ ${render_galaxy_repository_actions( repository )}
    Repair tool shed repository '${repository.name}'


    -
    + <% ordered_repo_info_dicts = repair_dict.get( 'ordered_repo_info_dicts', [] ) %> From 19ab93c29e7301c3a9b57058c16c7c7b30796db6 Mon Sep 17 00:00:00 2001 From: Carl Eberhard Date: Tue, 9 Jul 2013 12:10:59 -0400 Subject: [PATCH 253/318] Fix form url in history permissions form --- templates/webapps/galaxy/history/permissions.mako | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/webapps/galaxy/history/permissions.mako b/templates/webapps/galaxy/history/permissions.mako index e5c553e04c0..71d1099ea51 100644 --- a/templates/webapps/galaxy/history/permissions.mako +++ b/templates/webapps/galaxy/history/permissions.mako @@ -4,5 +4,6 @@ %if trans.user: <% history = trans.get_history() %> - ${render_permission_form( history, history.name, h.url_for(controller='root'), trans.user.all_roles() )} + ${render_permission_form( history, history.name, + h.url_for( controller='root', action='history_set_default_permissions' ), trans.user.all_roles() )} %endif From a82a893f44b104e8bf88a2e7d6660eabe7594530 Mon Sep 17 00:00:00 2001 From: Carl Eberhard Date: Tue, 9 Jul 2013 12:20:19 -0400 Subject: [PATCH 254/318] Fix admin left panel scrollbar --- templates/webapps/galaxy/admin/index.mako | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/templates/webapps/galaxy/admin/index.mako b/templates/webapps/galaxy/admin/index.mako index 41130efe2ea..6d82d685eee 100644 --- a/templates/webapps/galaxy/admin/index.mako +++ b/templates/webapps/galaxy/admin/index.mako @@ -17,6 +17,13 @@ #left { background: #C1C9E5 url(${h.url_for('/static/style/menu_bg.png')}) top repeat-x; } + + .unified-panel-body { + overflow: auto; + } + .toolMenu { + margin-left: 10px; + } @@ -36,7 +43,7 @@
    Administration
    -
    +
    Security
    From a4c1284ce107d4d62da746942d09a99471e057a7 Mon Sep 17 00:00:00 2001 From: Aysam Guerler Date: Tue, 9 Jul 2013 13:32:58 -0400 Subject: [PATCH 255/318] Scratchbook: Update, fix tooltip, re-factor code --- static/scripts/galaxy.frame.js | 95 ++++++++++--------- static/scripts/mvc/dataset/hda-base.js | 2 +- static/scripts/packed/galaxy.frame.js | 2 +- static/scripts/packed/mvc/dataset/hda-base.js | 2 +- static/style/blue/galaxy.frame.css | 5 + static/style/blue/galaxy.frame.masthead.css | 5 + templates/webapps/galaxy/base_panels.mako | 4 +- 7 files changed, 66 insertions(+), 49 deletions(-) diff --git a/static/scripts/galaxy.frame.js b/static/scripts/galaxy.frame.js index be84b5a6ced..99f0fca9f7c 100644 --- a/static/scripts/galaxy.frame.js +++ b/static/scripts/galaxy.frame.js @@ -123,8 +123,8 @@ var GalaxyFrameManager = Backbone.View.extend( this.event_initialize(); // add - //$(".galaxy-frame-active").tooltip({title: "Enable/Disable Scratchbook"}); - //$(".galaxy-frame-load").tooltip({title: "Show/Hide Scratchbook"}); + $(".galaxy-frame-active").tooltip({title: "Enable/Disable Scratchbook"}); + $(".galaxy-frame-load").tooltip({title: "Show/Hide Scratchbook"}); // catch window resize event var self = this; @@ -731,8 +731,7 @@ var GalaxyFrameManager = Backbone.View.extend( this.visible = false; // hide - for (var i in this.frame_list) - $(this.frame_list[i].id).fadeOut('fast'); + $(".galaxy-frame").fadeOut('fast'); // add class $(".galaxy-frame-load .icon").addClass("fa-icon-eye-close"); @@ -748,8 +747,7 @@ var GalaxyFrameManager = Backbone.View.extend( this.visible = true; // show - for (var i in this.frame_list) - $(this.frame_list[i].id).fadeIn('fast'); + $(".galaxy-frame").fadeIn('fast'); // add class $(".galaxy-frame-load .icon").addClass("fa-icon-eye-open"); @@ -800,59 +798,68 @@ var GalaxyFrameManager = Backbone.View.extend( if (!this.active) { // load frame in main window - if (options.center) + if (options.location == 'center') { var galaxy_main = $( window.parent.document ).find( 'iframe#galaxy_main' ); galaxy_main.attr( 'src', options.content ); } else window.location = options.content; - + // stop return; } - + + // check for number of frames + if (this.frame_counter > this.options.frame_max) + { + alert("You have reached the maximum number of allowed frames (" + this.options.frame_max + ")."); + return; + } + // generate frame identifier var frame_id = '#galaxy-frame-' + (this.frame_counter_id++); // check if frame exists - if ($(frame_id).length === 0 && this.options.frame_max > this.frame_counter) + if ($(frame_id).length !== 0) { - // reset top - this.top = this.options.top_min; - - // set dimensions - options.width = this.to_pixel_coord('width', this.options.frame.cols); - options.height = this.to_pixel_coord('height', this.options.frame.rows); - - // append - $(this.el).append(this.frame_template(frame_id.substring(1), options.title, options.type, options.content)); - - // construct a new frame - var frame = { - id : frame_id, - screen_location : {}, - grid_location : {}, - grid_rank : null, - grid_lock : false - }; + alert("This frame already exists. This page might contain multiple frame managers."); + return; + } + + // reset top + this.top = this.options.top_min; - // increase frame counter - this.frame_counter++; + // append + $(this.el).append(this.frame_template(frame_id.substring(1), options.title, options.type, options.content)); - // add to frame list - this.frame_list[frame_id] = frame; - - // resize - this.frame_resize(frame, {width: options.width, height: options.height}); + // construct a new frame + var frame = { + id : frame_id, + screen_location : {}, + grid_location : {}, + grid_rank : null, + grid_lock : false + }; + + // set dimensions + options.width = this.to_pixel_coord('width', this.options.frame.cols); + options.height = this.to_pixel_coord('height', this.options.frame.rows); + + // add to frame list + this.frame_list[frame_id] = frame; + + // increase frame counter + this.frame_counter++; + + // resize + this.frame_resize(frame, {width: options.width, height: options.height}); - // place frame - this.frame_insert(frame, {top: 0, left: 0}, true); + // place frame + this.frame_insert(frame, {top: 0, left: 0}, true); - // show frames if hidden - if (!this.visible) - this.panel_show_hide(); - } else - alert("You have reached the maximum number of allowed frames (" + this.options.frame_max + ")."); + // show frames if hidden + if (!this.visible) + this.panel_show_hide(); }, // frame insert at given location @@ -922,7 +929,7 @@ var GalaxyFrameManager = Backbone.View.extend( // panel menu this.menu_refresh(); }, - + // naive frame place frame_place: function(frame, animate) { @@ -1052,7 +1059,7 @@ var GalaxyFrameManager = Backbone.View.extend( '' + title + '' + '' + '' + - '
    ' + + '
    ' + '
    ' + content + '
    ' + '
    ' + diff --git a/static/scripts/mvc/dataset/hda-base.js b/static/scripts/mvc/dataset/hda-base.js index 792e4405304..9e3d0f4dc50 100644 --- a/static/scripts/mvc/dataset/hda-base.js +++ b/static/scripts/mvc/dataset/hda-base.js @@ -237,7 +237,7 @@ var HDABaseView = Backbone.View.extend( LoggableMixin ).extend( } else { displayBtnData.title = _l( 'View data' ); - displayBtnData.href = "javascript:parent.frame_manager.frame_new({title: 'Data Viewer', type: 'url', center: true, content: '" + this.urls.display + "'});"; + displayBtnData.href = "javascript:parent.frame_manager.frame_new({title: 'Data Viewer', type: 'url', location: 'center', content: '" + this.urls.display + "'});"; } this.displayButton = new IconButtonView({ model : new IconButton( displayBtnData ) }); diff --git a/static/scripts/packed/galaxy.frame.js b/static/scripts/packed/galaxy.frame.js index 6ab83f16be6..9072f9ece29 100644 --- a/static/scripts/packed/galaxy.frame.js +++ b/static/scripts/packed/galaxy.frame.js @@ -1 +1 @@ -define(["utils/galaxy.css","libs/backbone/backbone-relational"],function(b){var a=Backbone.View.extend({el:"#everything",el_header:"#masthead",options:{frame:{cols:6,rows:3},rows:1000,cell:130,margin:5,scroll:5,top_min:40,frame_max:10},cols:0,top:0,top_max:0,frame_counter:0,frame_counter_id:0,frame_list:[],galaxy_frame_shadow:null,visible:false,active:false,initialize:function(d){b.load_file(d.url.styles+"/galaxy.frame.css");if(d){this.options=_.defaults(d,this.options)}this.top=this.top_max=this.options.top_min;$(this.el).append(this.frame_template_background());$(this.el).append(this.frame_template_menu());$(this.el_header).append(this.frame_template_header());var e="#galaxy-frame-shadow";$(this.el).append(this.frame_template_shadow(e.substring(1)));this.galaxy_frame_shadow={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};this.frame_resize(this.galaxy_frame_shadow,{width:0,height:0});this.frame_list[e]=this.galaxy_frame_shadow;this.panel_refresh();this.event_initialize();var c=this;$(window).resize(function(){c.panel_refresh()});window.onbeforeunload=function(){if(c.frame_counter>0){return"You opened "+c.frame_counter+" frame(s) which will be lost."}}},is_mobile:function(){return navigator.userAgent.match(/mobile|(iPad)|(iPhone)|(iPod)|(android)|(webOS)/i)},event:{type:null,target:null,xy:null},event_initialize:function(){this.events={mousemove:"event_frame_mouse_move",mouseup:"event_frame_mouse_up",mouseleave:"event_frame_mouse_up",mousewheel:"event_panel_scroll",DOMMouseScroll:"event_panel_scroll","mousedown .galaxy-frame":"event_frame_mouse_down","mousedown .galaxy-frame-active":"event_panel_active","mousedown .galaxy-frame-load":"event_panel_load","mousedown .galaxy-frame-background":"event_panel_load","mousedown .galaxy-frame-scroll-up":"event_panel_scroll_up","mousedown .galaxy-frame-scroll-down":"event_panel_scroll_down","mousedown .f-close":"event_frame_close","mousedown .f-pin":"event_frame_lock"};this.delegateEvents(this.events)},event_frame_mouse_down:function(c){if(this.event.type!==null){return}if($(c.target).hasClass("f-header")||$(c.target).hasClass("f-title")){this.event.type="drag"}if($(c.target).hasClass("f-resize")){this.event.type="resize"}if(this.event.type===null){return}c.preventDefault();this.event.target=this.event_get_frame(c.target);if(this.event.target.grid_lock){this.event.type=null;return}this.event.xy={x:c.originalEvent.pageX,y:c.originalEvent.pageY};this.frame_drag_start(this.event.target)},event_frame_mouse_move:function(i){if(this.event.type!="drag"&&this.event.type!="resize"){return}var g={x:i.originalEvent.pageX,y:i.originalEvent.pageY};var d={x:g.x-this.event.xy.x,y:g.y-this.event.xy.y};this.event.xy=g;var h=this.frame_screen(this.event.target);if(this.event.type=="resize"){h.width+=d.x;h.height+=d.y;var f=this.options.cell-this.options.margin-1;h.width=Math.max(h.width,f);h.height=Math.max(h.height,f);this.frame_resize(this.event.target,h);h.width=this.to_grid_coord("width",h.width)+1;h.height=this.to_grid_coord("height",h.height)+1;h.width=this.to_pixel_coord("width",h.width);h.height=this.to_pixel_coord("height",h.height);this.frame_resize(this.galaxy_frame_shadow,h);this.frame_insert(this.galaxy_frame_shadow,{top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)})}if(this.event.type=="drag"){h.left+=d.x;h.top+=d.y;this.frame_offset(this.event.target,h);var c={top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)};if(c.left!==0){c.left++}this.frame_insert(this.galaxy_frame_shadow,c)}},event_frame_mouse_up:function(c){if(this.event.type!="drag"&&this.event.type!="resize"){return}this.frame_drag_stop(this.event.target);this.event.type=null},event_frame_close:function(d){if(this.event.type!==null){return}d.preventDefault();var f=this.event_get_frame(d.target);var c=this;$(f.id).fadeOut("fast",function(){$(f.id).remove();delete c.frame_list[f.id];c.frame_counter--;c.panel_refresh(true);c.panel_animation_complete();if(c.visible&&c.frame_counter==0){c.panel_show_hide()}})},event_frame_lock:function(c){if(this.event.type!==null){return}c.preventDefault();var d=this.event_get_frame(c.target);if(d.grid_lock){d.grid_lock=false;$(d.id).find(".f-pin").removeClass("f-toggle");$(d.id).find(".f-header").removeClass("f-not-allowed");$(d.id).find(".f-title").removeClass("f-not-allowed");$(d.id).find(".f-resize").show();$(d.id).find(".f-close").show()}else{d.grid_lock=true;$(d.id).find(".f-pin").addClass("f-toggle");$(d.id).find(".f-header").addClass("f-not-allowed");$(d.id).find(".f-title").addClass("f-not-allowed");$(d.id).find(".f-resize").hide();$(d.id).find(".f-close").hide()}},event_panel_load:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_show_hide()},event_panel_active:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_active_disable()},event_panel_scroll:function(c){if(this.event.type!==null||!this.visible){return}c.preventDefault();var d=c.originalEvent.detail?c.originalEvent.detail:c.originalEvent.wheelDelta/-3;this.panel_scroll(d)},event_panel_scroll_up:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_scroll(-this.options.scroll)},event_panel_scroll_down:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_scroll(this.options.scroll)},event_get_frame:function(c){return this.frame_list["#"+$(c).closest(".galaxy-frame").attr("id")]},frame_drag_start:function(d){this.frame_focus(d,true);var c=this.frame_screen(d);this.frame_resize(this.galaxy_frame_shadow,c);this.frame_grid(this.galaxy_frame_shadow,d.grid_location);d.grid_location=null;$(this.galaxy_frame_shadow.id).show();$(".f-cover").show()},frame_drag_stop:function(d){this.frame_focus(d,false);var c=this.frame_screen(this.galaxy_frame_shadow);this.frame_resize(d,c);this.frame_grid(d,this.galaxy_frame_shadow.grid_location,true);this.galaxy_frame_shadow.grid_location=null;$(this.galaxy_frame_shadow.id).hide();$(".f-cover").hide();this.panel_animation_complete()},to_grid_coord:function(e,d){var c=(e=="width"||e=="height")?1:-1;if(e=="top"){d-=this.top}return parseInt((d+c*this.options.margin)/this.options.cell,10)},to_pixel_coord:function(e,f){var c=(e=="width"||e=="height")?1:-1;var d=(f*this.options.cell)-c*this.options.margin;if(e=="top"){d+=this.top}return d},to_grid:function(c){return{top:this.to_grid_coord("top",c.top),left:this.to_grid_coord("left",c.left),width:this.to_grid_coord("width",c.width),height:this.to_grid_coord("height",c.height)}},to_pixel:function(c){return{top:this.to_pixel_coord("top",c.top),left:this.to_pixel_coord("left",c.left),width:this.to_pixel_coord("width",c.width),height:this.to_pixel_coord("height",c.height)}},is_collision:function(e){function c(h,g){return !(h.left>g.left+g.width-1||h.left+h.width-1g.top+g.height-1||h.top+h.height-1this.frame_counter){this.top=this.options.top_min;d.width=this.to_pixel_coord("width",this.options.frame.cols);d.height=this.to_pixel_coord("height",this.options.frame.rows);$(this.el).append(this.frame_template(e.substring(1),d.title,d.type,d.content));var f={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};this.frame_counter++;this.frame_list[e]=f;this.frame_resize(f,{width:d.width,height:d.height});this.frame_insert(f,{top:0,left:0},true);if(!this.visible){this.panel_show_hide()}}else{alert("You have reached the maximum number of allowed frames ("+this.options.frame_max+").")}},frame_insert:function(j,c,e){var d=[];if(j){j.grid_location=null;d.push([j,this.location_rank(c)])}var g=null;for(g in this.frame_list){var h=this.frame_list[g];if(h.grid_location!==null&&!h.grid_lock){h.grid_location=null;d.push([h,h.grid_rank])}}d.sort(function(k,f){var m=k[1];var l=f[1];return ml?1:0)});for(g=0;g'}return'
    '+e+'
    '+d+'
    '},frame_template_shadow:function(c){return'
    '},frame_template_background:function(){return'
    '},frame_template_header:function(){return'
    0
    '},frame_template_menu:function(){return'
    '}});return{GalaxyFrameManager:a}}); \ No newline at end of file +define(["utils/galaxy.css","libs/backbone/backbone-relational"],function(b){var a=Backbone.View.extend({el:"#everything",el_header:"#masthead",options:{frame:{cols:6,rows:3},rows:1000,cell:130,margin:5,scroll:5,top_min:40,frame_max:10},cols:0,top:0,top_max:0,frame_counter:0,frame_counter_id:0,frame_list:[],galaxy_frame_shadow:null,visible:false,active:false,initialize:function(d){b.load_file(d.url.styles+"/galaxy.frame.css");if(d){this.options=_.defaults(d,this.options)}this.top=this.top_max=this.options.top_min;$(this.el).append(this.frame_template_background());$(this.el).append(this.frame_template_menu());$(this.el_header).append(this.frame_template_header());var e="#galaxy-frame-shadow";$(this.el).append(this.frame_template_shadow(e.substring(1)));this.galaxy_frame_shadow={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};this.frame_resize(this.galaxy_frame_shadow,{width:0,height:0});this.frame_list[e]=this.galaxy_frame_shadow;this.panel_refresh();this.event_initialize();$(".galaxy-frame-active").tooltip({title:"Enable/Disable Scratchbook"});$(".galaxy-frame-load").tooltip({title:"Show/Hide Scratchbook"});var c=this;$(window).resize(function(){c.panel_refresh()});window.onbeforeunload=function(){if(c.frame_counter>0){return"You opened "+c.frame_counter+" frame(s) which will be lost."}}},is_mobile:function(){return navigator.userAgent.match(/mobile|(iPad)|(iPhone)|(iPod)|(android)|(webOS)/i)},event:{type:null,target:null,xy:null},event_initialize:function(){this.events={mousemove:"event_frame_mouse_move",mouseup:"event_frame_mouse_up",mouseleave:"event_frame_mouse_up",mousewheel:"event_panel_scroll",DOMMouseScroll:"event_panel_scroll","mousedown .galaxy-frame":"event_frame_mouse_down","mousedown .galaxy-frame-active":"event_panel_active","mousedown .galaxy-frame-load":"event_panel_load","mousedown .galaxy-frame-background":"event_panel_load","mousedown .galaxy-frame-scroll-up":"event_panel_scroll_up","mousedown .galaxy-frame-scroll-down":"event_panel_scroll_down","mousedown .f-close":"event_frame_close","mousedown .f-pin":"event_frame_lock"};this.delegateEvents(this.events)},event_frame_mouse_down:function(c){if(this.event.type!==null){return}if($(c.target).hasClass("f-header")||$(c.target).hasClass("f-title")){this.event.type="drag"}if($(c.target).hasClass("f-resize")){this.event.type="resize"}if(this.event.type===null){return}c.preventDefault();this.event.target=this.event_get_frame(c.target);if(this.event.target.grid_lock){this.event.type=null;return}this.event.xy={x:c.originalEvent.pageX,y:c.originalEvent.pageY};this.frame_drag_start(this.event.target)},event_frame_mouse_move:function(i){if(this.event.type!="drag"&&this.event.type!="resize"){return}var g={x:i.originalEvent.pageX,y:i.originalEvent.pageY};var d={x:g.x-this.event.xy.x,y:g.y-this.event.xy.y};this.event.xy=g;var h=this.frame_screen(this.event.target);if(this.event.type=="resize"){h.width+=d.x;h.height+=d.y;var f=this.options.cell-this.options.margin-1;h.width=Math.max(h.width,f);h.height=Math.max(h.height,f);this.frame_resize(this.event.target,h);h.width=this.to_grid_coord("width",h.width)+1;h.height=this.to_grid_coord("height",h.height)+1;h.width=this.to_pixel_coord("width",h.width);h.height=this.to_pixel_coord("height",h.height);this.frame_resize(this.galaxy_frame_shadow,h);this.frame_insert(this.galaxy_frame_shadow,{top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)})}if(this.event.type=="drag"){h.left+=d.x;h.top+=d.y;this.frame_offset(this.event.target,h);var c={top:this.to_grid_coord("top",h.top),left:this.to_grid_coord("left",h.left)};if(c.left!==0){c.left++}this.frame_insert(this.galaxy_frame_shadow,c)}},event_frame_mouse_up:function(c){if(this.event.type!="drag"&&this.event.type!="resize"){return}this.frame_drag_stop(this.event.target);this.event.type=null},event_frame_close:function(d){if(this.event.type!==null){return}d.preventDefault();var f=this.event_get_frame(d.target);var c=this;$(f.id).fadeOut("fast",function(){$(f.id).remove();delete c.frame_list[f.id];c.frame_counter--;c.panel_refresh(true);c.panel_animation_complete();if(c.visible&&c.frame_counter==0){c.panel_show_hide()}})},event_frame_lock:function(c){if(this.event.type!==null){return}c.preventDefault();var d=this.event_get_frame(c.target);if(d.grid_lock){d.grid_lock=false;$(d.id).find(".f-pin").removeClass("f-toggle");$(d.id).find(".f-header").removeClass("f-not-allowed");$(d.id).find(".f-title").removeClass("f-not-allowed");$(d.id).find(".f-resize").show();$(d.id).find(".f-close").show()}else{d.grid_lock=true;$(d.id).find(".f-pin").addClass("f-toggle");$(d.id).find(".f-header").addClass("f-not-allowed");$(d.id).find(".f-title").addClass("f-not-allowed");$(d.id).find(".f-resize").hide();$(d.id).find(".f-close").hide()}},event_panel_load:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_show_hide()},event_panel_active:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_active_disable()},event_panel_scroll:function(c){if(this.event.type!==null||!this.visible){return}c.preventDefault();var d=c.originalEvent.detail?c.originalEvent.detail:c.originalEvent.wheelDelta/-3;this.panel_scroll(d)},event_panel_scroll_up:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_scroll(-this.options.scroll)},event_panel_scroll_down:function(c){if(this.event.type!==null){return}c.preventDefault();this.panel_scroll(this.options.scroll)},event_get_frame:function(c){return this.frame_list["#"+$(c).closest(".galaxy-frame").attr("id")]},frame_drag_start:function(d){this.frame_focus(d,true);var c=this.frame_screen(d);this.frame_resize(this.galaxy_frame_shadow,c);this.frame_grid(this.galaxy_frame_shadow,d.grid_location);d.grid_location=null;$(this.galaxy_frame_shadow.id).show();$(".f-cover").show()},frame_drag_stop:function(d){this.frame_focus(d,false);var c=this.frame_screen(this.galaxy_frame_shadow);this.frame_resize(d,c);this.frame_grid(d,this.galaxy_frame_shadow.grid_location,true);this.galaxy_frame_shadow.grid_location=null;$(this.galaxy_frame_shadow.id).hide();$(".f-cover").hide();this.panel_animation_complete()},to_grid_coord:function(e,d){var c=(e=="width"||e=="height")?1:-1;if(e=="top"){d-=this.top}return parseInt((d+c*this.options.margin)/this.options.cell,10)},to_pixel_coord:function(e,f){var c=(e=="width"||e=="height")?1:-1;var d=(f*this.options.cell)-c*this.options.margin;if(e=="top"){d+=this.top}return d},to_grid:function(c){return{top:this.to_grid_coord("top",c.top),left:this.to_grid_coord("left",c.left),width:this.to_grid_coord("width",c.width),height:this.to_grid_coord("height",c.height)}},to_pixel:function(c){return{top:this.to_pixel_coord("top",c.top),left:this.to_pixel_coord("left",c.left),width:this.to_pixel_coord("width",c.width),height:this.to_pixel_coord("height",c.height)}},is_collision:function(e){function c(h,g){return !(h.left>g.left+g.width-1||h.left+h.width-1g.top+g.height-1||h.top+h.height-1this.options.frame_max){alert("You have reached the maximum number of allowed frames ("+this.options.frame_max+").");return}var e="#galaxy-frame-"+(this.frame_counter_id++);if($(e).length!==0){alert("This frame already exists. This page might contain multiple frame managers.");return}this.top=this.options.top_min;$(this.el).append(this.frame_template(e.substring(1),d.title,d.type,d.content));var f={id:e,screen_location:{},grid_location:{},grid_rank:null,grid_lock:false};d.width=this.to_pixel_coord("width",this.options.frame.cols);d.height=this.to_pixel_coord("height",this.options.frame.rows);this.frame_list[e]=f;this.frame_counter++;this.frame_resize(f,{width:d.width,height:d.height});this.frame_insert(f,{top:0,left:0},true);if(!this.visible){this.panel_show_hide()}},frame_insert:function(j,c,e){var d=[];if(j){j.grid_location=null;d.push([j,this.location_rank(c)])}var g=null;for(g in this.frame_list){var h=this.frame_list[g];if(h.grid_location!==null&&!h.grid_lock){h.grid_location=null;d.push([h,h.grid_rank])}}d.sort(function(k,f){var m=k[1];var l=f[1];return ml?1:0)});for(g=0;g'}return'
    '+e+'
    '+d+'
    '},frame_template_shadow:function(c){return'
    '},frame_template_background:function(){return'
    '},frame_template_header:function(){return'
    0
    '},frame_template_menu:function(){return'
    '}});return{GalaxyFrameManager:a}}); \ No newline at end of file diff --git a/static/scripts/packed/mvc/dataset/hda-base.js b/static/scripts/packed/mvc/dataset/hda-base.js index bc6b9d7357d..25fc3188665 100644 --- a/static/scripts/packed/mvc/dataset/hda-base.js +++ b/static/scripts/packed/mvc/dataset/hda-base.js @@ -1 +1 @@ -var HDABaseView=Backbone.View.extend(LoggableMixin).extend({tagName:"div",className:"historyItemContainer",initialize:function(a){if(a.logger){this.logger=this.model.logger=a.logger}this.log(this+".initialize:",a);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton];if(!a.urlTemplates){throw ("HDAView needs urlTemplates on initialize")}this.urlTemplates=a.urlTemplates;this.expanded=a.expanded||false;this.model.bind("change",function(c,d){var b=_.without(_.keys(d.changes),"display_apps","display_types");if(b.length){this.render()}else{if(this.expanded){this._render_displayApps()}}},this)},render:function(){var b=this,e=this.model.get("id"),c=this.model.get("state"),a=$("
    ").attr("id","historyItem-"+e),d=(this.$el.children().size()===0);this.$el.attr("id","historyItemContainer-"+e);this.urls=this._renderUrls(this.urlTemplates,this.model.toJSON());a.addClass("historyItemWrapper").addClass("historyItem").addClass("historyItem-"+c);a.append(this._render_warnings());a.append(this._render_titleBar());this._setUpBehaviors(a);this.body=$(this._render_body());a.append(this.body);this.$el.fadeOut("fast",function(){b.$el.children().remove();b.$el.append(a).fadeIn("fast",function(){b.log(b+" rendered:",b.$el);var f="rendered";if(d){f+=":initial"}else{if(b.model.inReadyState()){f+=":ready"}}b.trigger(f)})});return this},_renderUrls:function(d,a){var b=this,c={};_.each(d,function(e,f){if(_.isObject(e)){c[f]=b._renderUrls(e,a)}else{if(f==="meta_download"){c[f]=b._renderMetaDownloadUrls(e,a)}else{try{c[f]=_.template(e,a)}catch(g){throw (b+"._renderUrls error: "+g+"\n rendering:"+e+"\n with "+JSON.stringify(a))}}}});return c},_renderMetaDownloadUrls:function(b,a){return _.map(a.meta_files,function(c){return{url:_.template(b,{id:a.id,file_type:c.file_type}),file_type:c.file_type}})},_setUpBehaviors:function(a){a=a||this.$el;make_popup_menus(a);a.find(".tooltip").tooltip({placement:"bottom"})},_render_warnings:function(){return $(jQuery.trim(HDABaseView.templates.messages(this.model.toJSON())))},_render_titleBar:function(){var a=$('
    ');a.append(this._render_titleButtons());a.append('');a.append(this._render_titleLink());return a},_render_titleButtons:function(){var a=$('
    ');a.append(this._render_displayButton());return a},_render_displayButton:function(){if((this.model.get("state")===HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(this.model.get("state")===HistoryDatasetAssociation.STATES.NEW)||(!this.model.get("accessible"))){this.displayButton=null;return null}var a={icon_class:"display",target:"galaxy_main"};if(this.model.get("purged")){a.enabled=false;a.title=_l("Cannot display datasets removed from disk")}else{a.title=_l("View data");a.href="javascript:parent.frame_manager.frame_new({title: 'Data Viewer', type: 'url', center: true, content: '"+this.urls.display+"'});"}this.displayButton=new IconButtonView({model:new IconButton(a)});return this.displayButton.render().$el},_render_titleLink:function(){return $(jQuery.trim(HDABaseView.templates.titleLink(_.extend(this.model.toJSON(),{urls:this.urls}))))},_render_hdaSummary:function(){var a=_.extend(this.model.toJSON(),{urls:this.urls});return HDABaseView.templates.hdaSummary(a)},_render_primaryActionButtons:function(c){var a=this,b=$("
    ").attr("id","primary-actions-"+this.model.get("id"));_.each(c,function(d){b.append(d.call(a))});return b},_render_downloadButton:function(){if(this.model.get("purged")||!this.model.hasData()){return null}var a=HDABaseView.templates.downloadLinks(_.extend(this.model.toJSON(),{urls:this.urls}));return $(a.trim())},_render_showParamsButton:function(){this.showParamsButton=new IconButtonView({model:new IconButton({title:_l("View details"),href:this.urls.show_params,target:"galaxy_main",icon_class:"information"})});return this.showParamsButton.render().$el},_render_displayAppArea:function(){return $("
    ").addClass("display-apps")},_render_displayApps:function(c){c=c||this.$el;var d=c.find("div.display-apps"),a=this.model.get("display_types"),b=this.model.get("display_apps");if((!this.model.hasData())||(!c||!c.length)||(!d.length)){return}d.html(null);if(!_.isEmpty(a)){d.append(HDABaseView.templates.displayApps({displayApps:a}))}if(!_.isEmpty(b)){d.append(HDABaseView.templates.displayApps({displayApps:b}))}},_render_peek:function(){var a=this.model.get("peek");if(!a){return null}return $("
    ").append($("
    ").attr("id","peek"+this.model.get("id")).addClass("peek").append(a))},_render_body:function(){var a=$("
    ").attr("id","info-"+this.model.get("id")).addClass("historyItemBody").attr("style","display: none");if(this.expanded){this._render_body_html(a);a.css("display","block")}return a},_render_body_html:function(a){a.html("");switch(this.model.get("state")){case HistoryDatasetAssociation.STATES.NEW:this._render_body_new(a);break;case HistoryDatasetAssociation.STATES.NOT_VIEWABLE:this._render_body_not_viewable(a);break;case HistoryDatasetAssociation.STATES.UPLOAD:this._render_body_uploading(a);break;case HistoryDatasetAssociation.STATES.PAUSED:this._render_body_paused(a);break;case HistoryDatasetAssociation.STATES.QUEUED:this._render_body_queued(a);break;case HistoryDatasetAssociation.STATES.RUNNING:this._render_body_running(a);break;case HistoryDatasetAssociation.STATES.ERROR:this._render_body_error(a);break;case HistoryDatasetAssociation.STATES.DISCARDED:this._render_body_discarded(a);break;case HistoryDatasetAssociation.STATES.SETTING_METADATA:this._render_body_setting_metadata(a);break;case HistoryDatasetAssociation.STATES.EMPTY:this._render_body_empty(a);break;case HistoryDatasetAssociation.STATES.FAILED_METADATA:this._render_body_failed_metadata(a);break;case HistoryDatasetAssociation.STATES.OK:this._render_body_ok(a);break;default:a.append($('
    Error: unknown dataset state "'+this.model.get("state")+'".
    '))}a.append('
    ');this._setUpBehaviors(a)},_render_body_new:function(b){var a=_l("This is a new dataset and not all of its data are available yet");b.append($("
    "+_l(a)+"
    "))},_render_body_not_viewable:function(a){a.append($("
    "+_l("You do not have permission to view dataset")+"
    "))},_render_body_uploading:function(a){a.append($("
    "+_l("Dataset is uploading")+"
    "))},_render_body_queued:function(a){a.append($("
    "+_l("Job is waiting to run")+"
    "));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_paused:function(a){a.append($("
    "+_l("Job is paused.  Use the history menu to resume")+"
    "));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_running:function(a){a.append("
    "+_l("Job is currently running")+"
    ");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_error:function(a){if(!this.model.get("purged")){a.append($("
    "+this.model.get("misc_blurb")+"
    "))}a.append((_l("An error occurred with this dataset")+": "+$.trim(this.model.get("misc_info"))+""));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers.concat([this._render_downloadButton])))},_render_body_discarded:function(a){a.append("
    "+_l("The job creating this dataset was cancelled before completion")+".
    ");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_setting_metadata:function(a){a.append($("
    "+_l("Metadata is being auto-detected")+".
    "))},_render_body_empty:function(a){a.append($("
    "+_l("No data")+": "+this.model.get("misc_blurb")+"
    "));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_failed_metadata:function(a){a.append($(HDABaseView.templates.failedMetadata(_.extend(this.model.toJSON(),{urls:this.urls}))));this._render_body_ok(a)},_render_body_ok:function(a){a.append(this._render_hdaSummary());if(this.model.isDeletedOrPurged()){a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));return}a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));a.append('
    ');a.append(this._render_displayAppArea());this._render_displayApps(a);a.append(this._render_peek())},events:{"click .historyItemTitle":"toggleBodyVisibility"},toggleBodyVisibility:function(c,a){var b=this;this.expanded=(a===undefined)?(!this.body.is(":visible")):(a);if(this.expanded){b._render_body_html(b.body);this.body.slideDown("fast",function(){b.trigger("body-expanded",b.model.get("id"))})}else{this.body.slideUp("fast",function(){b.trigger("body-collapsed",b.model.get("id"))})}},remove:function(b){var a=this;this.$el.fadeOut("fast",function(){a.$el.remove();a.off();if(b){b()}})},toString:function(){var a=(this.model)?(this.model+""):("(no model)");return"HDABaseView("+a+")"}});HDABaseView.templates={warningMsg:Handlebars.templates["template-warningmessagesmall"],messages:Handlebars.templates["template-hda-warning-messages"],titleLink:Handlebars.templates["template-hda-titleLink"],hdaSummary:Handlebars.templates["template-hda-hdaSummary"],downloadLinks:Handlebars.templates["template-hda-downloadLinks"],failedMetadata:Handlebars.templates["template-hda-failedMetadata"],displayApps:Handlebars.templates["template-hda-displayApps"]};
\ No newline at end of file
+var HDABaseView=Backbone.View.extend(LoggableMixin).extend({tagName:"div",className:"historyItemContainer",initialize:function(a){if(a.logger){this.logger=this.model.logger=a.logger}this.log(this+".initialize:",a);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton];if(!a.urlTemplates){throw ("HDAView needs urlTemplates on initialize")}this.urlTemplates=a.urlTemplates;this.expanded=a.expanded||false;this.model.bind("change",function(c,d){var b=_.without(_.keys(d.changes),"display_apps","display_types");if(b.length){this.render()}else{if(this.expanded){this._render_displayApps()}}},this)},render:function(){var b=this,e=this.model.get("id"),c=this.model.get("state"),a=$("
    ").attr("id","historyItem-"+e),d=(this.$el.children().size()===0);this.$el.attr("id","historyItemContainer-"+e);this.urls=this._renderUrls(this.urlTemplates,this.model.toJSON());a.addClass("historyItemWrapper").addClass("historyItem").addClass("historyItem-"+c);a.append(this._render_warnings());a.append(this._render_titleBar());this._setUpBehaviors(a);this.body=$(this._render_body());a.append(this.body);this.$el.fadeOut("fast",function(){b.$el.children().remove();b.$el.append(a).fadeIn("fast",function(){b.log(b+" rendered:",b.$el);var f="rendered";if(d){f+=":initial"}else{if(b.model.inReadyState()){f+=":ready"}}b.trigger(f)})});return this},_renderUrls:function(d,a){var b=this,c={};_.each(d,function(e,f){if(_.isObject(e)){c[f]=b._renderUrls(e,a)}else{if(f==="meta_download"){c[f]=b._renderMetaDownloadUrls(e,a)}else{try{c[f]=_.template(e,a)}catch(g){throw (b+"._renderUrls error: "+g+"\n rendering:"+e+"\n with "+JSON.stringify(a))}}}});return c},_renderMetaDownloadUrls:function(b,a){return _.map(a.meta_files,function(c){return{url:_.template(b,{id:a.id,file_type:c.file_type}),file_type:c.file_type}})},_setUpBehaviors:function(a){a=a||this.$el;make_popup_menus(a);a.find(".tooltip").tooltip({placement:"bottom"})},_render_warnings:function(){return $(jQuery.trim(HDABaseView.templates.messages(this.model.toJSON())))},_render_titleBar:function(){var a=$('
    ');a.append(this._render_titleButtons());a.append('');a.append(this._render_titleLink());return a},_render_titleButtons:function(){var a=$('
    ');a.append(this._render_displayButton());return a},_render_displayButton:function(){if((this.model.get("state")===HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(this.model.get("state")===HistoryDatasetAssociation.STATES.NEW)||(!this.model.get("accessible"))){this.displayButton=null;return null}var a={icon_class:"display",target:"galaxy_main"};if(this.model.get("purged")){a.enabled=false;a.title=_l("Cannot display datasets removed from disk")}else{a.title=_l("View data");a.href="javascript:parent.frame_manager.frame_new({title: 'Data Viewer', type: 'url', location: 'center', content: '"+this.urls.display+"'});"}this.displayButton=new IconButtonView({model:new IconButton(a)});return this.displayButton.render().$el},_render_titleLink:function(){return $(jQuery.trim(HDABaseView.templates.titleLink(_.extend(this.model.toJSON(),{urls:this.urls}))))},_render_hdaSummary:function(){var a=_.extend(this.model.toJSON(),{urls:this.urls});return HDABaseView.templates.hdaSummary(a)},_render_primaryActionButtons:function(c){var a=this,b=$("
    ").attr("id","primary-actions-"+this.model.get("id"));_.each(c,function(d){b.append(d.call(a))});return b},_render_downloadButton:function(){if(this.model.get("purged")||!this.model.hasData()){return null}var a=HDABaseView.templates.downloadLinks(_.extend(this.model.toJSON(),{urls:this.urls}));return $(a.trim())},_render_showParamsButton:function(){this.showParamsButton=new IconButtonView({model:new IconButton({title:_l("View details"),href:this.urls.show_params,target:"galaxy_main",icon_class:"information"})});return this.showParamsButton.render().$el},_render_displayAppArea:function(){return $("
    ").addClass("display-apps")},_render_displayApps:function(c){c=c||this.$el;var d=c.find("div.display-apps"),a=this.model.get("display_types"),b=this.model.get("display_apps");if((!this.model.hasData())||(!c||!c.length)||(!d.length)){return}d.html(null);if(!_.isEmpty(a)){d.append(HDABaseView.templates.displayApps({displayApps:a}))}if(!_.isEmpty(b)){d.append(HDABaseView.templates.displayApps({displayApps:b}))}},_render_peek:function(){var a=this.model.get("peek");if(!a){return null}return $("
    ").append($("
    ").attr("id","peek"+this.model.get("id")).addClass("peek").append(a))},_render_body:function(){var a=$("
    ").attr("id","info-"+this.model.get("id")).addClass("historyItemBody").attr("style","display: none");if(this.expanded){this._render_body_html(a);a.css("display","block")}return a},_render_body_html:function(a){a.html("");switch(this.model.get("state")){case HistoryDatasetAssociation.STATES.NEW:this._render_body_new(a);break;case HistoryDatasetAssociation.STATES.NOT_VIEWABLE:this._render_body_not_viewable(a);break;case HistoryDatasetAssociation.STATES.UPLOAD:this._render_body_uploading(a);break;case HistoryDatasetAssociation.STATES.PAUSED:this._render_body_paused(a);break;case HistoryDatasetAssociation.STATES.QUEUED:this._render_body_queued(a);break;case HistoryDatasetAssociation.STATES.RUNNING:this._render_body_running(a);break;case HistoryDatasetAssociation.STATES.ERROR:this._render_body_error(a);break;case HistoryDatasetAssociation.STATES.DISCARDED:this._render_body_discarded(a);break;case HistoryDatasetAssociation.STATES.SETTING_METADATA:this._render_body_setting_metadata(a);break;case HistoryDatasetAssociation.STATES.EMPTY:this._render_body_empty(a);break;case HistoryDatasetAssociation.STATES.FAILED_METADATA:this._render_body_failed_metadata(a);break;case HistoryDatasetAssociation.STATES.OK:this._render_body_ok(a);break;default:a.append($('
    Error: unknown dataset state "'+this.model.get("state")+'".
    '))}a.append('
    ');this._setUpBehaviors(a)},_render_body_new:function(b){var a=_l("This is a new dataset and not all of its data are available yet");b.append($("
    "+_l(a)+"
    "))},_render_body_not_viewable:function(a){a.append($("
    "+_l("You do not have permission to view dataset")+"
    "))},_render_body_uploading:function(a){a.append($("
    "+_l("Dataset is uploading")+"
    "))},_render_body_queued:function(a){a.append($("
    "+_l("Job is waiting to run")+"
    "));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_paused:function(a){a.append($("
    "+_l("Job is paused.  Use the history menu to resume")+"
    "));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_running:function(a){a.append("
    "+_l("Job is currently running")+"
    ");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_error:function(a){if(!this.model.get("purged")){a.append($("
    "+this.model.get("misc_blurb")+"
    "))}a.append((_l("An error occurred with this dataset")+": "+$.trim(this.model.get("misc_info"))+""));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers.concat([this._render_downloadButton])))},_render_body_discarded:function(a){a.append("
    "+_l("The job creating this dataset was cancelled before completion")+".
    ");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_setting_metadata:function(a){a.append($("
    "+_l("Metadata is being auto-detected")+".
    "))},_render_body_empty:function(a){a.append($("
    "+_l("No data")+": "+this.model.get("misc_blurb")+"
    "));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_failed_metadata:function(a){a.append($(HDABaseView.templates.failedMetadata(_.extend(this.model.toJSON(),{urls:this.urls}))));this._render_body_ok(a)},_render_body_ok:function(a){a.append(this._render_hdaSummary());if(this.model.isDeletedOrPurged()){a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));return}a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));a.append('
    ');a.append(this._render_displayAppArea());this._render_displayApps(a);a.append(this._render_peek())},events:{"click .historyItemTitle":"toggleBodyVisibility"},toggleBodyVisibility:function(c,a){var b=this;this.expanded=(a===undefined)?(!this.body.is(":visible")):(a);if(this.expanded){b._render_body_html(b.body);this.body.slideDown("fast",function(){b.trigger("body-expanded",b.model.get("id"))})}else{this.body.slideUp("fast",function(){b.trigger("body-collapsed",b.model.get("id"))})}},remove:function(b){var a=this;this.$el.fadeOut("fast",function(){a.$el.remove();a.off();if(b){b()}})},toString:function(){var a=(this.model)?(this.model+""):("(no model)");return"HDABaseView("+a+")"}});HDABaseView.templates={warningMsg:Handlebars.templates["template-warningmessagesmall"],messages:Handlebars.templates["template-hda-warning-messages"],titleLink:Handlebars.templates["template-hda-titleLink"],hdaSummary:Handlebars.templates["template-hda-hdaSummary"],downloadLinks:Handlebars.templates["template-hda-downloadLinks"],failedMetadata:Handlebars.templates["template-hda-failedMetadata"],displayApps:Handlebars.templates["template-hda-displayApps"]};
\ No newline at end of file
diff --git a/static/style/blue/galaxy.frame.css b/static/style/blue/galaxy.frame.css
index 3e6c36c3d16..1faab583cc0 100644
--- a/static/style/blue/galaxy.frame.css
+++ b/static/style/blue/galaxy.frame.css
@@ -13,6 +13,11 @@
     color: #BCC800;
 }
 
+.bs-tooltip
+{
+    z-index: 34010;
+}
+
 /*
     galaxy-frames
 */
diff --git a/static/style/blue/galaxy.frame.masthead.css b/static/style/blue/galaxy.frame.masthead.css
index 485c34a6396..573f5e57eb5 100644
--- a/static/style/blue/galaxy.frame.masthead.css
+++ b/static/style/blue/galaxy.frame.masthead.css
@@ -14,6 +14,11 @@
 }
 
 #right
+{
+	top: 0 !important;
+}
+
+#left
 {
 	top: 0 !important;
 }
\ No newline at end of file
diff --git a/templates/webapps/galaxy/base_panels.mako b/templates/webapps/galaxy/base_panels.mako
index a317820bc7b..4ec54f01345 100644
--- a/templates/webapps/galaxy/base_panels.mako
+++ b/templates/webapps/galaxy/base_panels.mako
@@ -117,8 +117,8 @@ ${h.js( "mvc/base-mvc", "mvc/user/user-model", "mvc/user/user-quotameter" )}
     ${tab( "analysis", _("Analyze Data"), h.url_for( controller='/root', action='index' ) )}
     
     ## Workflow tab.
-    ${tab( "workflow", _("Workflow"), h.url_for( controller='/workflow', action='index' ) )}
-    
+    ${tab( "workflow", _("Workflow"), "javascript:frame_manager.frame_new({title: 'Workflow', type: 'url', content: '" + h.url_for( controller='/workflow', action='index' ) + "'});")}
+
     ## 'Shared Items' or Libraries tab.
     <%
         menu_options = [ 

From 2e67e5cce7a7cac301b7896c17007430a8ae2af6 Mon Sep 17 00:00:00 2001
From: Greg Von Kuster 
Date: Tue, 9 Jul 2013 14:13:12 -0400
Subject: [PATCH 256/318] Apply changes recently made to the left panel in
 Galaxy to the left panel in the Tool Shed.

---
 templates/webapps/tool_shed/admin/index.mako | 8 +++++++-
 templates/webapps/tool_shed/index.mako       | 7 ++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/templates/webapps/tool_shed/admin/index.mako b/templates/webapps/tool_shed/admin/index.mako
index f28396822a8..2c03f4fceab 100644
--- a/templates/webapps/tool_shed/admin/index.mako
+++ b/templates/webapps/tool_shed/admin/index.mako
@@ -1,7 +1,7 @@
 <%inherit file="/webapps/tool_shed/base_panels.mako"/>
 <%namespace file="/message.mako" import="render_msg" />
 
-<%def name="stylesheets()">   
+<%def name="stylesheets()">
     ## Include "base.css" for styling tool menu and forms (details)
     ${h.css( "base", "autocomplete_tagging", "tool_menu" )}
 
@@ -13,6 +13,12 @@
         #left {
             background: #C1C9E5 url(${h.url_for('/static/style/menu_bg.png')}) top repeat-x;
         }
+        .unified-panel-body {
+            overflow: auto;
+        }
+        .toolMenu {
+            margin-left: 10px;
+        }
     
 
 
diff --git a/templates/webapps/tool_shed/index.mako b/templates/webapps/tool_shed/index.mako
index 63d1335d948..dc132303885 100644
--- a/templates/webapps/tool_shed/index.mako
+++ b/templates/webapps/tool_shed/index.mako
@@ -2,7 +2,6 @@
 <%namespace file="/message.mako" import="render_msg" />
 
 <%def name="stylesheets()">
-    ${parent.stylesheets()}    
     ## Include "base.css" for styling tool menu and forms (details)
     ${h.css( "base", "autocomplete_tagging", "tool_menu" )}
 
@@ -14,6 +13,12 @@
         #left {
             background: #C1C9E5 url(${h.url_for('/static/style/menu_bg.png')}) top repeat-x;
         }
+        .unified-panel-body {
+            overflow: auto;
+        }
+        .toolMenu {
+            margin-left: 10px;
+        }
     
 
 

From 62db4e168d284aadeaa8bd37bd924725691b5fac Mon Sep 17 00:00:00 2001
From: Carl Eberhard 
Date: Tue, 9 Jul 2013 15:32:57 -0400
Subject: [PATCH 257/318] Fix unified-panel css in history/view.mako when used
 with/without 'use_panels'

---
 templates/webapps/galaxy/history/view.mako | 26 +++++++++++++---------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/templates/webapps/galaxy/history/view.mako b/templates/webapps/galaxy/history/view.mako
index 71a9c862189..9e9c4651aec 100644
--- a/templates/webapps/galaxy/history/view.mako
+++ b/templates/webapps/galaxy/history/view.mako
@@ -24,12 +24,9 @@
 <%def name="stylesheets()">
     ${parent.stylesheets()}
     ${h.css( "history", "autocomplete_tagging" )}
+
     
 
-    
 

From e5abce581db1c985f973ee35ea06985c7081421b Mon Sep 17 00:00:00 2001
From: Greg Von Kuster 
Date: Thu, 11 Jul 2013 14:59:27 -0400
Subject: [PATCH 260/318] Add support for categorizing repositories in the tool
 shed by type, with the first 2 supported types being "generic" and
 "tool_dependency_definition".  Repositories of type
 "tool_dependency_definition" can always contain only a single file named
 "tool_dependencies.xml", and the only metadata revision for these types of
 repositories will be the changelog tip.

---
 lib/galaxy/webapps/tool_shed/app.py           |  3 +
 .../tool_shed/controllers/repository.py       | 21 ++++++-
 .../webapps/tool_shed/model/__init__.py       | 36 ++++++++++--
 lib/galaxy/webapps/tool_shed/model/mapping.py |  1 +
 .../0020_add_repository_type_column.py        | 46 +++++++++++++++
 lib/tool_shed/repository_types/__init__.py    |  0
 lib/tool_shed/repository_types/generic.py     | 20 +++++++
 lib/tool_shed/repository_types/metadata.py    | 32 ++++++++++
 lib/tool_shed/repository_types/registry.py    | 17 ++++++
 .../tool_dependency_definition.py             | 43 ++++++++++++++
 lib/tool_shed/repository_types/util.py        | 30 ++++++++++
 lib/tool_shed/util/metadata_util.py           | 58 ++++++++++++++-----
 templates/webapps/tool_shed/admin/index.mako  |  2 +-
 templates/webapps/tool_shed/index.mako        |  2 +-
 .../repository/browse_repository.mako         |  1 +
 .../webapps/tool_shed/repository/common.mako  | 40 +++++++++++++
 .../repository/create_repository.mako         |  2 +
 .../repository/manage_repository.mako         |  1 +
 .../tool_shed/repository/rate_repository.mako |  1 +
 .../tool_shed/repository/view_repository.mako |  1 +
 20 files changed, 334 insertions(+), 23 deletions(-)
 create mode 100644 lib/galaxy/webapps/tool_shed/model/migrate/versions/0020_add_repository_type_column.py
 create mode 100644 lib/tool_shed/repository_types/__init__.py
 create mode 100644 lib/tool_shed/repository_types/generic.py
 create mode 100644 lib/tool_shed/repository_types/metadata.py
 create mode 100644 lib/tool_shed/repository_types/registry.py
 create mode 100644 lib/tool_shed/repository_types/tool_dependency_definition.py
 create mode 100644 lib/tool_shed/repository_types/util.py

diff --git a/lib/galaxy/webapps/tool_shed/app.py b/lib/galaxy/webapps/tool_shed/app.py
index 8d845b2150b..4e779bbe491 100644
--- a/lib/galaxy/webapps/tool_shed/app.py
+++ b/lib/galaxy/webapps/tool_shed/app.py
@@ -7,6 +7,7 @@ import galaxy.webapps.tool_shed.model
 from galaxy.openid.providers import OpenIDProviders
 from galaxy.web import security
 from galaxy.tags.tag_handler import CommunityTagHandler
+import tool_shed.repository_types.registry
 
 class UniverseApplication( object ):
     """Encapsulates the state of a Universe application"""
@@ -20,6 +21,8 @@ class UniverseApplication( object ):
         # Set up datatypes registry
         self.datatypes_registry = galaxy.datatypes.registry.Registry()
         self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config )
+        # Set up the repository_types registry.
+        self.repository_types_registry = tool_shed.repository_types.registry.Registry()
         # Determine the database url
         if self.config.database_connection:
             db_url = self.config.database_connection
diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py
index 8ee8bfcb1f5..0bd985aa771 100644
--- a/lib/galaxy/webapps/tool_shed/controllers/repository.py
+++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py
@@ -35,6 +35,7 @@ from galaxy.webapps.tool_shed.util import container_util
 import galaxy.tools
 import tool_shed.grids.repository_grids as repository_grids
 import tool_shed.grids.util as grids_util
+import tool_shed.repository_types.util as rt_util
 
 from galaxy import eggs
 eggs.require('mercurial')
@@ -665,10 +666,12 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
         # Update repository files for browsing.
         suc.update_repository( repo )
         metadata = self.get_metadata( trans, id, repository.tip( trans.app ) )
+        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans, repository=repository )
         return trans.fill_template( '/webapps/tool_shed/repository/browse_repository.mako',
                                     repository=repository,
                                     metadata=metadata,
                                     commit_message=commit_message,
+                                    repository_type_select_field=repository_type_select_field,
                                     message=message,
                                     status=status )
 
@@ -961,6 +964,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
         long_description = kwd.get( 'long_description', '' )
         category_ids = util.listify( kwd.get( 'category_id', '' ) )
         selected_categories = [ trans.security.decode_id( id ) for id in category_ids ]
+        repository_type = kwd.get( 'repository_type', rt_util.GENERIC )
         if kwd.get( 'create_repository_button', False ):
             error = False
             message = self.__validate_repository_name( name, trans.user )
@@ -974,6 +978,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
             else:
                 # Add the repository record to the db
                 repository = trans.app.model.Repository( name=name,
+                                                         type=repository_type,
                                                          description=description,
                                                          long_description=long_description,
                                                          user_id=trans.user.id )
@@ -1001,7 +1006,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
                 if category_ids:
                     # Create category associations
                     for category_id in category_ids:
-                        category = trans.sa_session.query(model.Category).get( trans.security.decode_id( category_id ) )
+                        category = trans.sa_session.query( model.Category ).get( trans.security.decode_id( category_id ) )
                         rca = trans.app.model.RepositoryCategoryAssociation( repository, category )
                         trans.sa_session.add( rca )
                         flush_needed = True
@@ -1012,12 +1017,14 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
                                                            action='view_repository',
                                                            message=message,
                                                            id=trans.security.encode_id( repository.id ) ) )
+        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans )
         return trans.fill_template( '/webapps/tool_shed/repository/create_repository.mako',
                                     name=name,
                                     description=description,
                                     long_description=long_description,
                                     selected_categories=selected_categories,
                                     categories=categories,
+                                    repository_type_select_field=repository_type_select_field,
                                     message=message,
                                     status=status )
 
@@ -1985,6 +1992,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
         message = kwd.get( 'message', ''  )
         status = kwd.get( 'status', 'done' )
         repository = suc.get_repository_in_tool_shed( trans, id )
+        repository_type = kwd.get( 'repository_type', str( repository.type ) )
         repo_dir = repository.repo_path( trans.app )
         repo = hg.repository( suc.get_configured_ui(), repo_dir )
         repo_name = kwd.get( 'repo_name', repository.name )
@@ -2016,6 +2024,9 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
                                                                   id=id,
                                                                   message=message,
                                                                   status='error' ) )
+            if repository_type != repository.type:
+                repository.type = repository_type
+                flush_needed = True
             if description != repository.description:
                 repository.description = description
                 flush_needed = True
@@ -2174,6 +2185,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
             else:
                 message += malicious_error
             status = 'error'
+        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans, repository=repository )
         malicious_check_box = CheckboxField( 'malicious', checked=is_malicious )
         skip_tool_tests_check_box = CheckboxField( 'skip_tool_tests', checked=skip_tool_tests_checked )
         categories = suc.get_categories( trans )
@@ -2202,6 +2214,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
                                     skip_tool_tests_check_box=skip_tool_tests_check_box,
                                     skip_tool_test=skip_tool_test,
                                     malicious_check_box=malicious_check_box,
+                                    repository_type_select_field=repository_type_select_field,
                                     message=message,
                                     status=status )
 
@@ -2352,6 +2365,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
         display_reviews = util.string_as_bool( kwd.get( 'display_reviews', False ) )
         rra = self.get_user_item_rating( trans.sa_session, trans.user, repository, webapp_model=trans.model )
         metadata = self.get_metadata( trans, id, repository.tip( trans.app ) )
+        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans, repository=repository )
         return trans.fill_template( '/webapps/tool_shed/repository/rate_repository.mako', 
                                     repository=repository,
                                     metadata=metadata,
@@ -2359,6 +2373,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
                                     display_reviews=display_reviews,
                                     num_ratings=num_ratings,
                                     rra=rra,
+                                    repository_type_select_field=repository_type_select_field,
                                     message=message,
                                     status=status )
 
@@ -2550,10 +2565,12 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
             else:
                 message = "Select at least 1 file to delete from the repository before clicking Delete selected files."
                 status = "error"
+        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans, repository=repository )
         return trans.fill_template( '/webapps/tool_shed/repository/browse_repository.mako',
                                     repo=repo,
                                     repository=repository,
                                     commit_message=commit_message,
+                                    repository_type_select_field=repository_type_select_field,
                                     message=message,
                                     status=status )
 
@@ -2974,6 +2991,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
                 message += malicious_error
             status = 'error'
         containers_dict = container_util.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata )
+        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans, repository=repository )
         return trans.fill_template( '/webapps/tool_shed/repository/view_repository.mako',
                                     repo=repo,
                                     repository=repository,
@@ -2987,6 +3005,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
                                     changeset_revision=changeset_revision,
                                     changeset_revision_select_field=changeset_revision_select_field,
                                     revision_label=revision_label,
+                                    repository_type_select_field=repository_type_select_field,
                                     message=message,
                                     status=status )
 
diff --git a/lib/galaxy/webapps/tool_shed/model/__init__.py b/lib/galaxy/webapps/tool_shed/model/__init__.py
index 4f505569abd..cf03a95b1cb 100644
--- a/lib/galaxy/webapps/tool_shed/model/__init__.py
+++ b/lib/galaxy/webapps/tool_shed/model/__init__.py
@@ -5,6 +5,7 @@ from galaxy import util
 from galaxy.util.bunch import Bunch
 from galaxy.util.hash_util import new_secure_hash
 from galaxy.model.item_attrs import APIItem
+import tool_shed.repository_types.util as rt_util
 
 from galaxy import eggs
 eggs.require( 'mercurial' )
@@ -130,18 +131,20 @@ class GalaxySession( object ):
 
 
 class Repository( object, APIItem ):
-    api_collection_visible_keys = ( 'id', 'name', 'description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' )
-    api_element_visible_keys = ( 'id', 'name', 'description', 'long_description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' )
+    api_collection_visible_keys = ( 'id', 'name', 'type', 'description', 'user_id', 'private', 'deleted', 'times_downloaded', 'deprecated' )
+    api_element_visible_keys = ( 'id', 'name', 'type', 'description', 'long_description', 'user_id', 'private', 'deleted', 'times_downloaded',
+                                 'deprecated' )
     file_states = Bunch( NORMAL = 'n',
                          NEEDS_MERGING = 'm',
                          MARKED_FOR_REMOVAL = 'r',
                          MARKED_FOR_ADDITION = 'a',
                          NOT_TRACKED = '?' )
 
-    def __init__( self, id=None, name=None, description=None, long_description=None, user_id=None, private=False, deleted=None, email_alerts=None,
-                  times_downloaded=0, deprecated=False ):
+    def __init__( self, id=None, name=None, type=None, description=None, long_description=None, user_id=None, private=False, deleted=None,
+                  email_alerts=None, times_downloaded=0, deprecated=False ):
         self.id = id
         self.name = name or "Unnamed repository"
+        self.type = type
         self.description = description
         self.long_description = long_description
         self.user_id = user_id
@@ -154,6 +157,24 @@ class Repository( object, APIItem ):
     def as_dict( self, value_mapper=None ):
         return self.get_api_value( view='element', value_mapper=value_mapper )
 
+    def can_change_type( self, app ):
+        # Allow changing the type only if the repository has no contents, has never been installed, or has never been changed from
+        # the default type.
+        if self.is_new( app ):
+            return True
+        if self.times_downloaded == 0:
+            return True
+        if self.type == rt_util.DEFAULT:
+            return True
+        return False
+
+    def can_change_type_to( self, app, new_type_label ):
+        if self.can_change_type( app ):
+            new_type = app.repository_types_registry.get_class_by_label( new_type_label )
+            if new_type.is_valid_for_type( self ):
+                return True
+        return False
+
     def get_api_value( self, view='collection', value_mapper=None ):
         if value_mapper is None:
             value_mapper = {}
@@ -173,6 +194,13 @@ class Repository( object, APIItem ):
             rval[ 'owner' ] = self.user.username
         return rval
 
+    def get_changesets_for_setting_metadata( self, app ):
+        type_class = self.get_type_class( app )
+        return type_class.get_changesets_for_setting_metadata( app, self )
+
+    def get_type_class( self, app ):
+        return app.repository_types_registry.get_class_by_label( self.type )
+
     def repo_path( self, app ):
         return app.hgweb_config_manager.get_entry( os.path.join( "repos", self.user.username, self.name ) )
 
diff --git a/lib/galaxy/webapps/tool_shed/model/mapping.py b/lib/galaxy/webapps/tool_shed/model/mapping.py
index 603585903ff..f8f0669ea90 100644
--- a/lib/galaxy/webapps/tool_shed/model/mapping.py
+++ b/lib/galaxy/webapps/tool_shed/model/mapping.py
@@ -106,6 +106,7 @@ Repository.table = Table( "repository", metadata,
     Column( "create_time", DateTime, default=now ),
     Column( "update_time", DateTime, default=now, onupdate=now ),
     Column( "name", TrimmedString( 255 ), index=True ),
+    Column( "type", TrimmedString( 255 ), index=True ),
     Column( "description" , TEXT ),
     Column( "long_description" , TEXT ),
     Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
diff --git a/lib/galaxy/webapps/tool_shed/model/migrate/versions/0020_add_repository_type_column.py b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0020_add_repository_type_column.py
new file mode 100644
index 00000000000..5810c12d7df
--- /dev/null
+++ b/lib/galaxy/webapps/tool_shed/model/migrate/versions/0020_add_repository_type_column.py
@@ -0,0 +1,46 @@
+"""Migration script to add the type column to the repository table."""
+
+from sqlalchemy import *
+from sqlalchemy.orm import *
+from migrate import *
+from migrate.changeset import *
+
+# Need our custom types, but don't import anything else from model
+from galaxy.model.custom_types import *
+
+import sys, logging
+log = logging.getLogger( __name__ )
+log.setLevel(logging.DEBUG)
+handler = logging.StreamHandler( sys.stdout )
+format = "%(name)s %(levelname)s %(asctime)s %(message)s"
+formatter = logging.Formatter( format )
+handler.setFormatter( formatter )
+log.addHandler( handler )
+
+metadata = MetaData()
+
+def upgrade( migrate_engine ):
+    print __doc__
+    metadata.bind = migrate_engine
+    metadata.reflect()
+    Repository_table = Table( "repository", metadata, autoload=True )
+    c = Column( "type", TrimmedString( 255 ), index=True )
+    try:
+        # Create
+        c.create( Repository_table, index_name="ix_repository_type" )
+        assert c is Repository_table.c.type
+    except Exception, e:
+        print "Adding type column to the repository table failed: %s" % str( e )
+    # Update the type column to have the default generic value.
+    cmd = "UPDATE repository SET type = 'generic'"
+    migrate_engine.execute( cmd )
+
+def downgrade( migrate_engine ):
+    metadata.bind = migrate_engine
+    metadata.reflect()
+    # Drop type column from repository table.
+    Repository_table = Table( "repository", metadata, autoload=True )
+    try:
+        Repository_table.c.type.drop()
+    except Exception, e:
+        print "Dropping column type from the repository table failed: %s" % str( e )
diff --git a/lib/tool_shed/repository_types/__init__.py b/lib/tool_shed/repository_types/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/lib/tool_shed/repository_types/generic.py b/lib/tool_shed/repository_types/generic.py
new file mode 100644
index 00000000000..80856458c33
--- /dev/null
+++ b/lib/tool_shed/repository_types/generic.py
@@ -0,0 +1,20 @@
+import logging
+import tool_shed.repository_types.util as rt_util
+from tool_shed.repository_types.metadata import Metadata
+
+log = logging.getLogger( __name__ )
+
+
+class Generic( Metadata ):
+
+    def __init__( self ):
+        self.type = rt_util.GENERIC
+        self.label = 'Generic'
+
+    def is_valid_for_type( self, app, repository, revisions_to_check=None ):
+        """A repository's type can only be changed to the generic type if it is new or has never been installed."""
+        if repository.is_new( app ):
+            return True
+        if repository.times_downloaded == 0:
+            return True
+        return False
diff --git a/lib/tool_shed/repository_types/metadata.py b/lib/tool_shed/repository_types/metadata.py
new file mode 100644
index 00000000000..e63152a125a
--- /dev/null
+++ b/lib/tool_shed/repository_types/metadata.py
@@ -0,0 +1,32 @@
+import logging
+import tool_shed.repository_types.util as rt_util
+
+from galaxy import eggs
+eggs.require( 'mercurial' )
+from mercurial import hg
+from mercurial import ui
+
+log = logging.getLogger( __name__ )
+
+
+class Metadata( object ):
+
+    def __init__( self ):
+        self.type = None
+
+    def get_changesets_for_setting_metadata( self, app, repository ):
+        repo = hg.repository( ui.ui(), repository.repo_path( app ) )
+        return repo.changelog
+
+    def is_valid_for_type( self, repository, revisions_to_check=None ):
+        raise "Unimplemented Method"
+
+
+class TipOnly( Metadata ):
+
+    def __init__( self ):
+        self.type = None
+
+    def get_changesets_for_setting_metadata( self, app, repository ):
+        repo = hg.repository( ui.ui(), repository.repo_path( app ) )
+        return [ repo.changelog.tip() ]
diff --git a/lib/tool_shed/repository_types/registry.py b/lib/tool_shed/repository_types/registry.py
new file mode 100644
index 00000000000..353a84eff28
--- /dev/null
+++ b/lib/tool_shed/repository_types/registry.py
@@ -0,0 +1,17 @@
+import logging
+import generic
+import tool_dependency_definition
+from galaxy.util.odict import odict
+
+log = logging.getLogger( __name__ )
+
+
+class Registry( object ):
+
+    def __init__( self ):
+        self.repository_types_by_label = odict()
+        self.repository_types_by_label[ 'generic' ] = generic.Generic()
+        self.repository_types_by_label[ 'tool_dependency_definition' ] = tool_dependency_definition.ToolDependencyDefinition()
+
+    def get_class_by_label( self, label ):
+        return self.repository_types_by_label.get( label, None )
diff --git a/lib/tool_shed/repository_types/tool_dependency_definition.py b/lib/tool_shed/repository_types/tool_dependency_definition.py
new file mode 100644
index 00000000000..b7bf53b9305
--- /dev/null
+++ b/lib/tool_shed/repository_types/tool_dependency_definition.py
@@ -0,0 +1,43 @@
+import logging
+from tool_shed.repository_types.metadata import TipOnly
+import tool_shed.repository_types.util as rt_util
+import tool_shed.util.shed_util_common as suc
+
+from galaxy import eggs
+eggs.require( 'mercurial' )
+from mercurial import hg
+from mercurial import ui
+
+log = logging.getLogger( __name__ )
+
+class ToolDependencyDefinition( TipOnly ):
+
+    def __init__( self ):
+        self.type = rt_util.TOOL_DEPENDENCY_DEFINITION
+        self.label = 'Tool dependency definition'
+        self.valid_file_names = [ 'tool_dependencies.xml' ]
+
+    def is_valid_for_type( self, app, repository, revisions_to_check=None ):
+        """
+        Inspect the received repository's contents to determine if they abide by the rules defined for the contents of this type.
+        If the received revisions_to_check is a list of changeset revisions, then inspection will be restricted to the revisions
+        in the list.
+        """
+        repo = hg.repository( ui.ui(), repository.repo_path( app ) )
+        if revisions_to_check:
+            changeset_revisions = revisions_to_check
+        else:
+            changeset_revisions = repo.changelog
+        for changeset in changeset_revisions:
+            changeset_revision = str( repo.changectx( changeset ) )
+            ctx = repo.changectx( changeset )
+            # Inspect all files in the changeset (in sorted order) to make sure there is only one and it is named tool_dependencies.xml.
+            files_changed_in_changeset = ctx.files()
+            if len( files_changed_in_changeset ) > 1:
+                return False
+            for file_path in files_changed_in_changeset:
+                file_name = suc.strip_path( file_path )
+                if file_name not in self.valid_file_names:
+                    return False
+        return True
+        
\ No newline at end of file
diff --git a/lib/tool_shed/repository_types/util.py b/lib/tool_shed/repository_types/util.py
new file mode 100644
index 00000000000..dea4a1a1909
--- /dev/null
+++ b/lib/tool_shed/repository_types/util.py
@@ -0,0 +1,30 @@
+import logging
+from galaxy.web.form_builder import SelectField
+
+log = logging.getLogger( __name__ )
+
+GENERIC = 'generic'
+TOOL_DEPENDENCY_DEFINITION = 'tool_dependency_definition'
+
+types = [ GENERIC, TOOL_DEPENDENCY_DEFINITION ]
+
+def build_repository_type_select_fiels( trans, repository=None, name='repository_type' ):
+    """Called from the Tool Shed to generate the current list of supported repository types."""
+    if repository:
+        selected_type = str( repository.type )
+    else:
+        selected_type = None
+    repository_type_select_fiels = SelectField( name=name )
+    for type_label, type_class in trans.app.repository_types_registry.repository_types_by_label.items():
+        option_label = str( type_class.label )
+        option_value = str( type_class.type )
+        if selected_type and selected_type == option_value:
+            selected = True
+        else:
+            selected = False
+        if repository:
+            if type_class.is_valid_for_type( trans.app, repository ):
+                repository_type_select_fiels.add_option( option_label, option_value, selected=selected )
+        else:
+            repository_type_select_fiels.add_option( option_label, option_value, selected=selected )
+    return repository_type_select_fiels
diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py
index fbfa8f22c6a..9554b973720 100644
--- a/lib/tool_shed/util/metadata_util.py
+++ b/lib/tool_shed/util/metadata_util.py
@@ -12,6 +12,7 @@ from galaxy.util import json
 from galaxy.web import url_for
 from galaxy.webapps.tool_shed.util import container_util
 import tool_shed.util.shed_util_common as suc
+from tool_shed.repository_types.metadata import TipOnly
 from tool_shed.util import common_util
 from tool_shed.util import common_install_util
 from tool_shed.util import readme_util
@@ -65,7 +66,7 @@ def clean_repository_metadata( trans, id, changeset_revisions ):
             trans.sa_session.delete( repository_metadata )
             trans.sa_session.flush()
 
-def compare_changeset_revisions( ancestor_changeset_revision, ancestor_metadata_dict, current_changeset_revision, current_metadata_dict ):
+def compare_changeset_revisions( trans, ancestor_changeset_revision, ancestor_metadata_dict, current_changeset_revision, current_metadata_dict ):
     """Compare the contents of two changeset revisions to determine if a new repository metadata revision should be created."""
     # The metadata associated with ancestor_changeset_revision is ancestor_metadata_dict.  This changeset_revision is an ancestor of
     # current_changeset_revision which is associated with current_metadata_dict.  A new repository_metadata record will be created only
@@ -104,11 +105,11 @@ def compare_changeset_revisions( ancestor_changeset_revision, ancestor_metadata_
     # Uncomment the following if we decide that README files should affect how installable repository revisions are defined.  See the NOTE in the
     # compare_readme_files() method.
     # readme_file_comparision = compare_readme_files( ancestor_readme_files, current_readme_files )
-    repository_dependency_comparison = compare_repository_dependencies( ancestor_repository_dependencies, current_repository_dependencies )
-    tool_dependency_comparison = compare_tool_dependencies( ancestor_tool_dependencies, current_tool_dependencies )
-    workflow_comparison = compare_workflows( ancestor_workflows, current_workflows )
-    datatype_comparison = compare_datatypes( ancestor_datatypes, current_datatypes )
-    data_manager_comparison = compare_data_manager( ancestor_data_manager, current_data_manager )
+    repository_dependency_comparison = compare_repository_dependencies( trans, ancestor_repository_dependencies, current_repository_dependencies )
+    tool_dependency_comparison = compare_tool_dependencies( trans, ancestor_tool_dependencies, current_tool_dependencies )
+    workflow_comparison = compare_workflows( trans, ancestor_workflows, current_workflows )
+    datatype_comparison = compare_datatypes( trans, ancestor_datatypes, current_datatypes )
+    data_manager_comparison = compare_data_manager( trans, ancestor_data_manager, current_data_manager )
     # Handle case where all metadata is the same.
     if ancestor_guids == current_guids and \
         repository_dependency_comparison == EQUAL and \
@@ -134,7 +135,7 @@ def compare_changeset_revisions( ancestor_changeset_revision, ancestor_metadata_
             return SUBSET
     return NOT_EQUAL_AND_NOT_SUBSET
 
-def compare_data_manager( ancestor_metadata, current_metadata ):
+def compare_data_manager( trans, ancestor_metadata, current_metadata ):
     """Determine if ancestor_metadata is the same as or a subset of current_metadata for data_managers."""
     def __data_manager_dict_to_tuple_list( metadata_dict ):
         # we do not check tool_guid or tool conf file name
@@ -149,7 +150,7 @@ def compare_data_manager( ancestor_metadata, current_metadata ):
         return SUBSET
     return NOT_EQUAL_AND_NOT_SUBSET
 
-def compare_datatypes( ancestor_datatypes, current_datatypes ):
+def compare_datatypes( trans, ancestor_datatypes, current_datatypes ):
     """Determine if ancestor_datatypes is the same as or a subset of current_datatypes."""
     # Each datatype dict looks something like: {"dtype": "galaxy.datatypes.images:Image", "extension": "pdf", "mimetype": "application/pdf"}
     if len( ancestor_datatypes ) <= len( current_datatypes ):
@@ -193,7 +194,7 @@ def compare_readme_files( ancestor_readme_files, current_readme_files ):
             return SUBSET
     return NOT_EQUAL_AND_NOT_SUBSET
 
-def compare_repository_dependencies( ancestor_repository_dependencies, current_repository_dependencies ):
+def compare_repository_dependencies( trans, ancestor_repository_dependencies, current_repository_dependencies ):
     """Determine if ancestor_repository_dependencies is the same as or a subset of current_repository_dependencies."""
     # The list of repository_dependencies looks something like: [["http://localhost:9009", "emboss_datatypes", "test", "ab03a2a5f407", False]].
     # Create a string from each tuple in the list for easier comparison.
@@ -211,14 +212,18 @@ def compare_repository_dependencies( ancestor_repository_dependencies, current_r
                     found_in_current = True
                     break
             if not found_in_current:
-                return NOT_EQUAL_AND_NOT_SUBSET
+                # In some cases, the only difference between a dependency definition in the lists is the changeset_revision value.  We'll
+                # check to see if this is the case, and if the defined dependency is a repository that has metadata set only on it's tip.        
+                if not different_revision_defines_tip_only_repository_dependency( ancestor_tup, current_repository_dependencies ):
+                    return NOT_EQUAL_AND_NOT_SUBSET
+                return SUBSET
         if len( ancestor_repository_dependencies ) == len( current_repository_dependencies ):
             return EQUAL
         else:
             return SUBSET
     return NOT_EQUAL_AND_NOT_SUBSET
 
-def compare_tool_dependencies( ancestor_tool_dependencies, current_tool_dependencies ):
+def compare_tool_dependencies( trans, ancestor_tool_dependencies, current_tool_dependencies ):
     """Determine if ancestor_tool_dependencies is the same as or a subset of current_tool_dependencies."""
     # The tool_dependencies dictionary looks something like:
     # {'bwa/0.5.9': {'readme': 'some string', 'version': '0.5.9', 'type': 'package', 'name': 'bwa'}}
@@ -238,7 +243,7 @@ def compare_tool_dependencies( ancestor_tool_dependencies, current_tool_dependen
             return SUBSET
     return NOT_EQUAL_AND_NOT_SUBSET
 
-def compare_workflows( ancestor_workflows, current_workflows ):
+def compare_workflows( trans, ancestor_workflows, current_workflows ):
     """Determine if ancestor_workflows is the same as current_workflows or if ancestor_workflows is a subset of current_workflows."""
     if len( ancestor_workflows ) <= len( current_workflows ):
         for ancestor_workflow_tup in ancestor_workflows:
@@ -341,6 +346,21 @@ def create_or_update_repository_metadata( trans, id, repository, changeset_revis
                 break
     return repository_metadata
 
+def different_revision_defines_tip_only_repository_dependency( rd_tup, repository_dependencies ):
+    """
+    Determine if the only difference between rd_tup and a dependency definition in the list of repository_dependencies is the changeset_revision value.
+    """
+    new_metadata_required = False
+    rd_tool_shed, rd_name, rd_owner, rd_changeset_revision, rd_prior_installation_required = suc.parse_repository_dependency_tuple( rd_tup )
+    for repository_dependency in repository_dependencies:
+        tool_shed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( repository_dependency )
+        if rd_tool_shed == tool_shed and rd_name == name and rd_owner == owner:
+            # Determine if the repository represented by the dependency tuple is an instance of the repository type TipOnly.
+            required_repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
+            repository_type_class = trans.app.repository_types_registry.get_class_by_label( required_repository.type )
+            return isinstance( repository_type_class, TipOnly )
+    return False
+
 def generate_data_manager_metadata( app, repository, repo_dir, data_manager_config_filename, metadata_dict, shed_config_dict=None ):
     """Update the received metadata_dict with information from the parsed data_manager_config_filename."""
     if data_manager_config_filename is None:
@@ -1308,7 +1328,10 @@ def new_repository_dependency_metadata_required( trans, repository_metadata, met
                 # The saved metadata must be a subset of the new metadata.
                 for saved_repository_dependency in saved_repository_dependencies:
                     if saved_repository_dependency not in new_repository_dependencies:
-                        return True
+                        # In some cases, the only difference between a dependency definition in the lists is the changeset_revision value.  We'll
+                        # check to see if this is the case, and if the defined dependency is a repository that has metadata set only on it's tip.
+                        if not different_revision_defines_tip_only_repository_dependency( saved_repository_dependency, new_repository_dependencies ):
+                            return True
                 return False
             else:
                 # The repository_dependencies.xml file must have been deleted, so create a new repository_metadata record so we always have
@@ -1588,7 +1611,7 @@ def reset_all_metadata_on_repository_in_tool_shed( trans, id ):
     ancestor_metadata_dict = None
     invalid_file_tups = []
     home_dir = os.getcwd()
-    for changeset in repo.changelog:
+    for changeset in repository.get_changesets_for_setting_metadata( trans.app ):
         work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-ramorits" )
         current_changeset_revision = str( repo.changectx( changeset ) )
         ctx = repo.changectx( changeset )
@@ -1619,7 +1642,8 @@ def reset_all_metadata_on_repository_in_tool_shed( trans, id ):
                     # EQUAL - ancestor metadata is equivalent to current metadata, so continue from current
                     # SUBSET - ancestor metadata is a subset of current metadata, so continue from current
                     # NOT_EQUAL_AND_NOT_SUBSET - ancestor metadata is neither equal to nor a subset of current metadata, so persist ancestor metadata.
-                    comparison = compare_changeset_revisions( ancestor_changeset_revision,
+                    comparison = compare_changeset_revisions( trans,
+                                                              ancestor_changeset_revision,
                                                               ancestor_metadata_dict,
                                                               current_changeset_revision,
                                                               current_metadata_dict )
@@ -1752,7 +1776,9 @@ def set_repository_metadata( trans, repository, content_alert_str='', **kwd ):
                                                                                  persist=False )
     if metadata_dict:
         repository_metadata = None
-        if new_metadata_required_for_utilities( trans, repository, metadata_dict ):
+        repository_type_class = trans.app.repository_types_registry.get_class_by_label( repository.type )
+        tip_only = isinstance( repository_type_class, TipOnly )
+        if not tip_only and new_metadata_required_for_utilities( trans, repository, metadata_dict ):
             # Create a new repository_metadata table row.
             repository_metadata = create_or_update_repository_metadata( trans, encoded_id, repository, repository.tip( trans.app ), metadata_dict )
             # If this is the first record stored for this repository, see if we need to send any email alerts.
diff --git a/templates/webapps/tool_shed/admin/index.mako b/templates/webapps/tool_shed/admin/index.mako
index 2c03f4fceab..7a8e45843c2 100644
--- a/templates/webapps/tool_shed/admin/index.mako
+++ b/templates/webapps/tool_shed/admin/index.mako
@@ -46,7 +46,7 @@
     
    
         
    Administration
    
     
    
-    
    
+    
    
         
    
             
    
                 
    
diff --git a/templates/webapps/tool_shed/index.mako b/templates/webapps/tool_shed/index.mako
index dc132303885..6e215901c96 100644
--- a/templates/webapps/tool_shed/index.mako
+++ b/templates/webapps/tool_shed/index.mako
@@ -46,7 +46,7 @@
     
    
         
    ${trans.app.shed_counter.valid_tools | h} valid tools on ${trans.app.shed_counter.generation_time | h}
    
     
    
-    
    
+    
    
         
    
             
    
                 %if user_id or repository_id:
diff --git a/templates/webapps/tool_shed/repository/browse_repository.mako b/templates/webapps/tool_shed/repository/browse_repository.mako
index ba6a022409a..df5dacb4e56 100644
--- a/templates/webapps/tool_shed/repository/browse_repository.mako
+++ b/templates/webapps/tool_shed/repository/browse_repository.mako
@@ -46,6 +46,7 @@ ${render_tool_shed_repository_actions( repository )}
                 ${render_clone_str( repository )}
             
    
         %endif
+        ${render_repository_type_select_field( repository_type_select_field, render_help=False )}
         %if can_push:
             
                 
    
diff --git a/templates/webapps/tool_shed/repository/common.mako b/templates/webapps/tool_shed/repository/common.mako
index d924a7bcb1e..3874c793052 100644
--- a/templates/webapps/tool_shed/repository/common.mako
+++ b/templates/webapps/tool_shed/repository/common.mako
@@ -163,6 +163,46 @@
     
 
 
+<%def name="render_repository_type_select_field( repository_type_select_field, render_help=True )">
+    
    
+        
+        <%
+            from tool_shed.repository_types import util
+            options = repository_type_select_field.options
+            repository_types = []
+            for option_tup in options:
+                repository_types.append( option_tup[ 1 ] )
+            render_as_text = len( options ) == 1
+            if render_as_text:
+                repository_type = options[ 0 ][ 0 ]
+        %>
+        %if render_as_text:
+            ${repository_type | h}
+            %if render_help:
+                
    
+                    This repository's type cannot be changed because it's contents are valid only for it's current type or it has been cloned.
+                
    
+            %endif
+        %else:
+            ${repository_type_select_field.get_html()}
+            %if render_help:
+                
    
+                    Select the repository type based on the following criteria.
+                    
      
+                        %if util.GENERIC in repository_types:
+                            
    • Generic - contents can be any set of valid Galaxy utilities
+                        %endif
+                        %if util.TOOL_DEPENDENCY_DEFINITION in repository_types:
+                            
    • Tool dependency definition - contents will always be restricted to one file named tool_dependencies.xml
+                        %endif
+                    
    
+                
    
+            %endif
+        %endif
+        
    
+    
    
+         
+            
 <%def name="render_sharable_str( repository, changeset_revision=None )">
     <%
         from tool_shed.util.shed_util_common import generate_sharable_link_for_repository_in_tool_shed
diff --git a/templates/webapps/tool_shed/repository/create_repository.mako b/templates/webapps/tool_shed/repository/create_repository.mako
index 6becc5f1b12..d26bafea1fa 100644
--- a/templates/webapps/tool_shed/repository/create_repository.mako
+++ b/templates/webapps/tool_shed/repository/create_repository.mako
@@ -1,5 +1,6 @@
 <%inherit file="/base.mako"/>
 <%namespace file="/message.mako" import="render_msg" />
+<%namespace file="/webapps/tool_shed/repository/common.mako" import="render_repository_type_select_field" />
 
 <%def name="javascripts()">
     ${parent.javascripts()}
@@ -23,6 +24,7 @@
                 
                 
    
             
    
+            ${render_repository_type_select_field( repository_type_select_field, render_help=True )}
             
    
                 
                 
diff --git a/templates/webapps/tool_shed/repository/manage_repository.mako b/templates/webapps/tool_shed/repository/manage_repository.mako
index e1786f56b4c..3d44aed7344 100644
--- a/templates/webapps/tool_shed/repository/manage_repository.mako
+++ b/templates/webapps/tool_shed/repository/manage_repository.mako
@@ -151,6 +151,7 @@ ${render_tool_shed_repository_actions( repository, metadata=metadata, changeset_
                 
    
                 
    
             
    
+            ${render_repository_type_select_field( repository_type_select_field, render_help=True )}
             
    
                 
                 
diff --git a/templates/webapps/tool_shed/repository/rate_repository.mako b/templates/webapps/tool_shed/repository/rate_repository.mako
index 77c48bc90a0..a7059fe0543 100644
--- a/templates/webapps/tool_shed/repository/rate_repository.mako
+++ b/templates/webapps/tool_shed/repository/rate_repository.mako
@@ -44,6 +44,7 @@ ${render_tool_shed_repository_actions( repository, repo=None, metadata=None, cha
                 ${render_clone_str( repository )}
             
    
         %endif
+        ${render_repository_type_select_field( repository_type_select_field, render_help=False )}
         
    
             
    
                 
diff --git a/templates/webapps/tool_shed/repository/view_repository.mako b/templates/webapps/tool_shed/repository/view_repository.mako
index 73b2f826aed..286e4995006 100644
--- a/templates/webapps/tool_shed/repository/view_repository.mako
+++ b/templates/webapps/tool_shed/repository/view_repository.mako
@@ -100,6 +100,7 @@
                 ${repository.name | h}
             %endif
         
    
+        ${render_repository_type_select_field( repository_type_select_field, render_help=False )}
         
    
             
             ${repository.description | h}

From 12aaa17023c39a511864103ead1d6bc1613a1779 Mon Sep 17 00:00:00 2001
From: Greg Von Kuster 
Date: Thu, 11 Jul 2013 15:16:08 -0400
Subject: [PATCH 261/318] Fix for generating the repository type select list,
 and fix a related typo in my last commit.

---
 .../webapps/tool_shed/controllers/repository.py    | 12 ++++++------
 lib/tool_shed/repository_types/util.py             | 14 ++++++++------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py
index 0bd985aa771..7b2d30f13e6 100644
--- a/lib/galaxy/webapps/tool_shed/controllers/repository.py
+++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py
@@ -666,7 +666,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
         # Update repository files for browsing.
         suc.update_repository( repo )
         metadata = self.get_metadata( trans, id, repository.tip( trans.app ) )
-        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans, repository=repository )
+        repository_type_select_field = rt_util.build_repository_type_select_field( trans, repository=repository )
         return trans.fill_template( '/webapps/tool_shed/repository/browse_repository.mako',
                                     repository=repository,
                                     metadata=metadata,
@@ -1017,7 +1017,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
                                                            action='view_repository',
                                                            message=message,
                                                            id=trans.security.encode_id( repository.id ) ) )
-        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans )
+        repository_type_select_field = rt_util.build_repository_type_select_field( trans )
         return trans.fill_template( '/webapps/tool_shed/repository/create_repository.mako',
                                     name=name,
                                     description=description,
@@ -2185,7 +2185,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
             else:
                 message += malicious_error
             status = 'error'
-        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans, repository=repository )
+        repository_type_select_field = rt_util.build_repository_type_select_field( trans, repository=repository )
         malicious_check_box = CheckboxField( 'malicious', checked=is_malicious )
         skip_tool_tests_check_box = CheckboxField( 'skip_tool_tests', checked=skip_tool_tests_checked )
         categories = suc.get_categories( trans )
@@ -2365,7 +2365,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
         display_reviews = util.string_as_bool( kwd.get( 'display_reviews', False ) )
         rra = self.get_user_item_rating( trans.sa_session, trans.user, repository, webapp_model=trans.model )
         metadata = self.get_metadata( trans, id, repository.tip( trans.app ) )
-        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans, repository=repository )
+        repository_type_select_field = rt_util.build_repository_type_select_field( trans, repository=repository )
         return trans.fill_template( '/webapps/tool_shed/repository/rate_repository.mako', 
                                     repository=repository,
                                     metadata=metadata,
@@ -2565,7 +2565,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
             else:
                 message = "Select at least 1 file to delete from the repository before clicking Delete selected files."
                 status = "error"
-        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans, repository=repository )
+        repository_type_select_field = rt_util.build_repository_type_select_field( trans, repository=repository )
         return trans.fill_template( '/webapps/tool_shed/repository/browse_repository.mako',
                                     repo=repo,
                                     repository=repository,
@@ -2991,7 +2991,7 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
                 message += malicious_error
             status = 'error'
         containers_dict = container_util.build_repository_containers_for_tool_shed( trans, repository, changeset_revision, repository_dependencies, repository_metadata )
-        repository_type_select_field = rt_util.build_repository_type_select_fiels( trans, repository=repository )
+        repository_type_select_field = rt_util.build_repository_type_select_field( trans, repository=repository )
         return trans.fill_template( '/webapps/tool_shed/repository/view_repository.mako',
                                     repo=repo,
                                     repository=repository,
diff --git a/lib/tool_shed/repository_types/util.py b/lib/tool_shed/repository_types/util.py
index dea4a1a1909..f7d4b2fbe3a 100644
--- a/lib/tool_shed/repository_types/util.py
+++ b/lib/tool_shed/repository_types/util.py
@@ -8,13 +8,13 @@ TOOL_DEPENDENCY_DEFINITION = 'tool_dependency_definition'
 
 types = [ GENERIC, TOOL_DEPENDENCY_DEFINITION ]
 
-def build_repository_type_select_fiels( trans, repository=None, name='repository_type' ):
+def build_repository_type_select_field( trans, repository=None, name='repository_type' ):
     """Called from the Tool Shed to generate the current list of supported repository types."""
     if repository:
         selected_type = str( repository.type )
     else:
         selected_type = None
-    repository_type_select_fiels = SelectField( name=name )
+    repository_type_select_field = SelectField( name=name )
     for type_label, type_class in trans.app.repository_types_registry.repository_types_by_label.items():
         option_label = str( type_class.label )
         option_value = str( type_class.type )
@@ -23,8 +23,10 @@ def build_repository_type_select_fiels( trans, repository=None, name='repository
         else:
             selected = False
         if repository:
-            if type_class.is_valid_for_type( trans.app, repository ):
-                repository_type_select_fiels.add_option( option_label, option_value, selected=selected )
+            if repository.type == option_value:
+                repository_type_select_field.add_option( option_label, option_value, selected=selected )
+            elif type_class.is_valid_for_type( trans.app, repository ):
+                repository_type_select_field.add_option( option_label, option_value, selected=selected )
         else:
-            repository_type_select_fiels.add_option( option_label, option_value, selected=selected )
-    return repository_type_select_fiels
+            repository_type_select_field.add_option( option_label, option_value, selected=selected )
+    return repository_type_select_field

From 441fc26e1e9f12ad644d3312b20e0e52220e3477 Mon Sep 17 00:00:00 2001
From: Greg Von Kuster 
Date: Thu, 11 Jul 2013 22:06:48 -0400
Subject: [PATCH 262/318] Contribution from Bjorn Gruning allowing installation
 of a repository to progress even if a repository dependency is not available.
  This will handle the case where a repository depnedency's env.sh file is not
 available due to an error in it's installation, but if possible, the
 dependent repository will proceed with it's installation.

---
 .../tool_dependencies/fabric_util.py          | 25 +++++++++++++------
 .../tool_dependencies/install_util.py         | 21 +++++++++-------
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py
index 19bd4378bd4..87f4ab8e777 100644
--- a/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py
+++ b/lib/tool_shed/galaxy_install/tool_dependencies/fabric_util.py
@@ -238,8 +238,11 @@ def install_and_build_package( app, tool_dependency, actions_dict ):
                             # in the set_environment action.
                             cmds = []
                             for env_shell_file_path in env_shell_file_paths:
-                                for i, env_setting in enumerate( open( env_shell_file_path ) ):
-                                    cmds.append( env_setting.strip( '\n' ) )
+                                if os.path.exists( env_shell_file_path ):
+                                    for env_setting in open( env_shell_file_path ):
+                                        cmds.append( env_setting.strip( '\n' ) )
+                                else:
+                                    log.debug( 'Invalid file %s specified, ignoring set_environment action.', env_shell_file_path )
                             env_var_dicts = action_dict[ 'environment_variable' ]
                             for env_var_dict in env_var_dicts:
                                 # Check for the presence of the $ENV[] key string and populate it if possible.
@@ -298,8 +301,11 @@ def install_and_build_package( app, tool_dependency, actions_dict ):
                             with settings( warn_only=True ):
                                 cmd = ''
                                 for env_shell_file_path in env_shell_file_paths:
-                                    for env_setting in open( env_shell_file_path ):
-                                        cmd += '%s\n' % env_setting
+                                    if os.path.exists( env_shell_file_path ):
+                                        for env_setting in open( env_shell_file_path ):
+                                            cmd += '%s\n' % env_setting
+                                    else:
+                                        log.debug( 'Invalid file %s specified, ignoring shell_command action.', env_shell_file_path )
                                 cmd += action_dict[ 'command' ]
                                 return_code = handle_command( app, tool_dependency, install_dir, cmd )
                                 if return_code:
@@ -307,10 +313,13 @@ def install_and_build_package( app, tool_dependency, actions_dict ):
                         elif action_type == 'template_command':
                             env_vars = dict()
                             for env_shell_file_path in env_shell_file_paths:
-                                for env_setting in open( env_shell_file_path ):
-                                    env_string = env_setting.split( ';' )[ 0 ]
-                                    env_name, env_path = env_string.split( '=' )
-                                    env_vars[ env_name ] = env_path
+                                if os.path.exists( env_shell_file_path ):
+                                    for env_setting in open( env_shell_file_path ):
+                                        env_string = env_setting.split( ';' )[ 0 ]
+                                        env_name, env_path = env_string.split( '=' )
+                                        env_vars[ env_name ] = env_path
+                                else:
+                                    log.debug( 'Invalid file %s specified, ignoring template_command action.', env_shell_file_path )
                             env_vars.update( common_util.get_env_var_values( install_dir ) )
                             language = action_dict[ 'language' ]
                             with settings( warn_only=True, **env_vars ):
diff --git a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py
index 2707eb04af5..2fe08a19fc7 100644
--- a/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py
+++ b/lib/tool_shed/galaxy_install/tool_dependencies/install_util.py
@@ -177,15 +177,18 @@ def handle_set_environment_entry_for_package( app, install_dir, tool_shed_reposi
                                                                            tool_dependency_name=package_name,
                                                                            tool_dependency_version=package_version )
                         env_sh_file_path = os.path.join( env_sh_file_dir, 'env.sh' )
-                        for i, line in enumerate( open( env_sh_file_path, 'r' ) ):
-                            env_var_dict = env_var_dicts[ i ]
-                            action = env_var_dict.get( 'action', None )
-                            name = env_var_dict.get( 'name', None )
-                            value = env_var_dict.get( 'value', None )
-                            if action and name and value:
-                                new_value = parse_env_shell_entry( action, name, value, line )
-                                env_var_dict[ 'value' ] = new_value
-                            new_env_var_dicts.append( env_var_dict )
+                        if os.path.exists( env_sh_file_path ):
+                            for i, line in enumerate( open( env_sh_file_path, 'r' ) ):
+                                env_var_dict = env_var_dicts[ i ]
+                                action = env_var_dict.get( 'action', None )
+                                name = env_var_dict.get( 'name', None )
+                                value = env_var_dict.get( 'value', None )
+                                if action and name and value:
+                                    new_value = parse_env_shell_entry( action, name, value, line )
+                                    env_var_dict[ 'value' ] = new_value
+                                new_env_var_dicts.append( env_var_dict )
+                        else:
+                            log.debug( 'Invalid file %s specified, ignoring set_environment_for_install action.', env_sh_file_path )
                         action_dict[ 'environment_variable' ] = new_env_var_dicts
                     else:
                         action_dict[ 'environment_variable' ] = env_var_dicts

From 2ecf3ea100befdaec57ebd19d9d1390549793496 Mon Sep 17 00:00:00 2001
From: Greg Von Kuster 
Date: Thu, 11 Jul 2013 22:24:35 -0400
Subject: [PATCH 263/318] Contribution from Bjorn Gruning to extend the valid
 list of readme files in tool shed repositories to include .rst files.

---
 lib/tool_shed/util/metadata_util.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py
index 9554b973720..d520bc45443 100644
--- a/lib/tool_shed/util/metadata_util.py
+++ b/lib/tool_shed/util/metadata_util.py
@@ -982,10 +982,11 @@ def get_parent_id( trans, id, old_id, version, guid, changeset_revisions ):
         return old_id
 
 def get_readme_file_names( repository_name ):
+    """Creates a list of valid filenames."""
     readme_files = [ 'readme', 'read_me', 'install' ]
-    valid_filenames = [ r for r in readme_files ]
-    for r in readme_files:
-        valid_filenames.append( '%s.txt' % r )
+    valid_filenames = map( lambda f: '%s.txt' % f, readme_files )
+    valid_filenames.extend( map( lambda f: '%s.rst' % f, readme_files ) )
+    valid_filenames.extend( readme_files )
     valid_filenames.append( '%s.txt' % repository_name )
     return valid_filenames
 

From baa51efee167b38810e476b52ab71133a7b8ed21 Mon Sep 17 00:00:00 2001
From: Greg Von Kuster 
Date: Thu, 11 Jul 2013 22:33:17 -0400
Subject: [PATCH 264/318] Contribution from Bjorn Gruning that renders readme
 files as restructured text. in the tool shed.

---
 templates/webapps/tool_shed/repository/common.mako | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/templates/webapps/tool_shed/repository/common.mako b/templates/webapps/tool_shed/repository/common.mako
index 3874c793052..864da61e3b7 100644
--- a/templates/webapps/tool_shed/repository/common.mako
+++ b/templates/webapps/tool_shed/repository/common.mako
@@ -600,7 +600,7 @@
 
 <%def name="render_readme( readme, pad, parent, row_counter )">
     <%
-        from tool_shed.util.shed_util_common import to_safe_string
+        from galaxy.util import rst_to_html
         encoded_id = trans.security.encode_id( readme.id )
     %>
    NameOwnerChangeset revision
    - +
    @@ -573,18 +563,8 @@ parent="${parent}" %endif id="libraryItem-${encoded_id}"> -
    Tool id: ${failed_test.tool_id | h}
    Tool version: ${failed_test.tool_id | h}
    Test: ${failed_test.test_id | h}
    - +
    @@ -703,6 +683,19 @@ %> +<%def name="render_tool_test_results_css()"> + + + <%def name="render_tool_dependency_installation_error( installation_error, pad, parent, row_counter, row_is_header=False )"> <% encoded_id = trans.security.encode_id( installation_error.id ) %> -
    Tool id: ${missing_test_component.tool_id | h}
    Tool version: ${missing_test_component.tool_version | h}
    Tool guid: ${missing_test_component.tool_guid | h}
    - +
    @@ -748,18 +731,8 @@ parent="${parent}" %endif id="libraryItem-${encoded_id}"> -
    TypeNameVersion
    - +
    %if not is_current_repository: @@ -789,18 +762,8 @@ parent="${parent}" %endif id="libraryItem-${encoded_id}"> - @@ -818,18 +781,8 @@ parent="${parent}" %endif id="libraryItem-${encoded_id}"> -
    Tool shedNameOwnerChangeset revision - +
    ${not_tested.reason | h}
    		 - +
    @@ -1206,6 +1159,7 @@ %endif %if tool_test_results_root_folder: + ${render_tool_test_results_css()}

    Automated tool test results
    From a1714c4ad257fae121f57775a56099d7be2746ae Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Mon, 29 Jul 2013 17:36:39 -0400 Subject: [PATCH 311/318] Add and document welcome_url config option. --- universe_wsgi.ini.sample | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index 64218309091..3fb91ae04d4 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -228,6 +228,10 @@ paste.app_factory = galaxy.web.buildapp:app_factory # -- Display sites +# The URL of the page to display in Galaxy's middle pane when loaded. This can be +# an absolute or relative URL. +#welcome_url = /static/welcome.html + # Galaxy can display data at various external browsers. These options specify # which browsers should be available. URLs and builds available at these # browsers are defined in the specifield files. From 8944c1ac3067cebb0df4b6bf69f5d419aa9e20f3 Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Mon, 29 Jul 2013 22:38:17 -0400 Subject: [PATCH 312/318] Move welcome_url setting to 'UI Localization' section of the config file. --- universe_wsgi.ini.sample | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index 3fb91ae04d4..0bed34a3b66 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -228,10 +228,6 @@ paste.app_factory = galaxy.web.buildapp:app_factory # -- Display sites -# The URL of the page to display in Galaxy's middle pane when loaded. This can be -# an absolute or relative URL. -#welcome_url = /static/welcome.html - # Galaxy can display data at various external browsers. These options specify # which browsers should be available. URLs and builds available at these # browsers are defined in the specifield files. @@ -284,6 +280,10 @@ nglims_config_file = tool-data/nglims.yaml # Append "/{brand}" to the "Galaxy" text in the masthead. #brand = None +# The URL of the page to display in Galaxy's middle pane when loaded. This can be +# an absolute or relative URL. +#welcome_url = /static/welcome.html + # The URL linked by the "Galaxy/brand" text. #logo_url = / From fdc46aac916a98e9569de95a709f59246598ac63 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 30 Jul 2013 09:23:39 -0400 Subject: [PATCH 313/318] Clear tool test results when the skip tool tests option is enabled. --- lib/galaxy/webapps/tool_shed/controllers/repository.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index f94d9e4fa5c..0543990b8bf 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1947,6 +1947,10 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): # Handle the mapper behavior. skip_tool_test = skip_tool_test[ 0 ] if skip_tool_tests_checked: + if repository_metadata.tool_test_results: + repository_metadata.tool_test_results = None + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() if skip_tool_test: comment = skip_tool_test.comment if comment != skip_tool_tests_comment: @@ -1959,12 +1963,12 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): comment=skip_tool_tests_comment ) trans.sa_session.add( skip_tool_test ) trans.sa_session.flush() - message = "Tools in this revision will be tested by the automated test framework." + message = "Tools in this revision will not be tested by the automated test framework." else: if skip_tool_test: trans.sa_session.delete( skip_tool_test ) trans.sa_session.flush() - message = "Tools in this revision will not be tested by the automated test framework." + message = "Tools in this revision will be tested by the automated test framework." elif kwd.get( 'manage_categories_button', False ): flush_needed = False # Delete all currently existing categories. From 3a71ed6ebcf97ac1128ecdbb51042944ad1f6a3b Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 30 Jul 2013 10:32:31 -0400 Subject: [PATCH 314/318] Fix for revision select on repositories with multiple installable revisions. --- lib/tool_shed/util/shed_util_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index afa21ed9cc4..9e3bef88dcf 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -815,7 +815,7 @@ def get_repository_from_refresh_on_change( trans, **kwd ): changeset_revision_str = 'changeset_revision_' if k.startswith( changeset_revision_str ): repository_id = trans.security.encode_id( int( k.lstrip( changeset_revision_str ) ) ) - repository = suc.get_repository_in_tool_shed( trans, repository_id ) + repository = get_repository_in_tool_shed( trans, repository_id ) if repository.tip( trans.app ) != v: return v, repository # This should never be reached - raise an exception? From 0f32b50f977e83c6f0436dd3efc428db684035e6 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 30 Jul 2013 12:31:59 -0400 Subject: [PATCH 315/318] Close branch. From 812aebdd62afc32714d65501ae09a1c700dae67a Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 30 Jul 2013 12:44:05 -0400 Subject: [PATCH 316/318] Backout af20b15f7eda, these changes will be reapplied after tools using the sam_fa_indices data table have been migrated to the Tool Shed in their original state and updated to use the new table format. --- tool-data/sam_fa_new_indices.loc.sample | 30 ------------------------- tool_data_table_conf.xml.sample | 20 ++--------------- 2 files changed, 2 insertions(+), 48 deletions(-) delete mode 100644 tool-data/sam_fa_new_indices.loc.sample diff --git a/tool-data/sam_fa_new_indices.loc.sample b/tool-data/sam_fa_new_indices.loc.sample deleted file mode 100644 index ddff8d4c46a..00000000000 --- a/tool-data/sam_fa_new_indices.loc.sample +++ /dev/null @@ -1,30 +0,0 @@ -#This is a sample file distributed with Galaxy that enables tools -#to use a directory of Samtools indexed sequences data files. You will need -#to create these data files and then create a sam_fa_new_indices.loc file -#similar to this one (store it in this directory) that points to -#the directories in which those files are stored. The sam_fa_new_indices.loc -#file has this format (white space characters are TAB characters): -# -# -# -#So, for example, if you had hg19 Canonical indexed stored in -# -# /depot/data2/galaxy/hg19/sam/, -# -#then the sam_fa_new_indices.loc entry would look like this: -# -#hg19canon hg19 Human (Homo sapiens): hg19 Canonical /depot/data2/galaxy/hg19/sam/hg19canon.fa -# -#and your /depot/data2/galaxy/hg19/sam/ directory -#would contain hg19canon.fa and hg19canon.fa.fai files. -# -#Your sam_fa_new_indices.loc file should include an entry per line for -#each index set you have stored. The file in the path does actually -#exist, but it should never be directly used. Instead, the name serves -#as a prefix for the index file. For example: -# -#hg18canon hg18 Human (Homo sapiens): hg18 Canonical /depot/data2/galaxy/hg18/sam/hg18canon.fa -#hg18full hg18 Human (Homo sapiens): hg18 Full /depot/data2/galaxy/hg18/sam/hg18full.fa -#hg19canon hg19 Human (Homo sapiens): hg19 Canonical /depot/data2/galaxy/hg19/sam/hg19canon.fa -#hg19full hg19 Human (Homo sapiens): hg19 Full /depot/data2/galaxy/hg19/sam/hg19full.fa - diff --git a/tool_data_table_conf.xml.sample b/tool_data_table_conf.xml.sample index b120b685e57..5ea328f1bc6 100644 --- a/tool_data_table_conf.xml.sample +++ b/tool_data_table_conf.xml.sample @@ -55,26 +55,10 @@ value, name, path
    Tool id: ${passed_test.tool_id | h}
    Tool version: ${passed_test.tool_id | h}
    Test: ${passed_test.test_id | h}
    - + line_type, value, path - - If you are updating your tool_data_table_conf.xml to the current - version you should first migrate your - tool-data/sam_fa_indices.loc file to a new - tool-data/sam_fa_new_indices.loc file with the format specified - below, which is explained in the relative sample file - tool-data/sam_fa_new_indices.loc.sample . - By using the new format it is possible to let the user choose - among multiple indexed genome variants having the same dbkey, - e.g. hg19canon vs. hg19full variants for hg19 dbkey. - --> -
    - value, dbkey, name, path -
    From 900cf1a98bb84710253b0143821f26264b936895 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 30 Jul 2013 12:50:29 -0400 Subject: [PATCH 317/318] See previous commit message. Note that cufflinks reverts to 0.0.6, not 0.0.7, due to a version increase that came after the changes that are being rolled back. --- tools/ngs_rna/cuffcompare_wrapper.py | 30 ++++++++++++++++++++++----- tools/ngs_rna/cuffcompare_wrapper.xml | 17 ++++++--------- tools/ngs_rna/cuffdiff_wrapper.py | 30 ++++++++++++++++++++++----- tools/ngs_rna/cuffdiff_wrapper.xml | 17 ++++++--------- tools/ngs_rna/cufflinks_wrapper.py | 30 ++++++++++++++++++++++----- tools/ngs_rna/cufflinks_wrapper.xml | 25 +++++++++------------- tools/ngs_rna/cuffmerge_wrapper.py | 30 ++++++++++++++++++++++----- tools/ngs_rna/cuffmerge_wrapper.xml | 17 ++++++--------- 8 files changed, 128 insertions(+), 68 deletions(-) diff --git a/tools/ngs_rna/cuffcompare_wrapper.py b/tools/ngs_rna/cuffcompare_wrapper.py index 9d9ad373abd..dba53ec434e 100644 --- a/tools/ngs_rna/cuffcompare_wrapper.py +++ b/tools/ngs_rna/cuffcompare_wrapper.py @@ -8,6 +8,20 @@ def stop_err( msg ): sys.stderr.write( '%s\n' % msg ) sys.exit() +# Copied from sam_to_bam.py: +def check_seq_file( dbkey, cached_seqs_pointer_file ): + seq_path = '' + for line in open( cached_seqs_pointer_file ): + line = line.rstrip( '\r\n' ) + if line and not line.startswith( '#' ) and line.startswith( 'index' ): + fields = line.split( '\t' ) + if len( fields ) < 3: + continue + if fields[1] == dbkey: + seq_path = fields[2].strip() + break + return seq_path + def __main__(): #Parse Command Line parser = optparse.OptionParser() @@ -16,7 +30,8 @@ def __main__(): parser.add_option( '-s', dest='use_seq_data', action="store_true", help='Causes cuffcompare to look into for fasta files with the underlying genomic sequences (one file per contig) against which your reads were aligned for some optional classification functions. For example, Cufflinks transcripts consisting mostly of lower-case bases are classified as repeats. Note that must contain one fasta file per reference chromosome, and each file must be named after the chromosome, and have a .fa or .fasta extension.') # Wrapper / Galaxy options. - parser.add_option( '', '--index', dest='index', help='The path of the reference genome' ) + parser.add_option( '', '--dbkey', dest='dbkey', help='The build of the reference dataset' ) + parser.add_option( '', '--index_dir', dest='index_dir', help='GALAXY_DATA_INDEX_DIR' ) parser.add_option( '', '--ref_file', dest='ref_file', help='The reference dataset from the history' ) # Outputs. @@ -45,16 +60,21 @@ def __main__(): # Set/link to sequence file. if options.use_seq_data: - if options.ref_file: + if options.ref_file != 'None': # Sequence data from history. # Create symbolic link to ref_file so that index will be created in working directory. seq_path = "ref.fa" os.symlink( options.ref_file, seq_path ) else: # Sequence data from loc file. - if not os.path.exists( options.index ): - stop_err( 'Reference genome %s not present, request it by reporting this error.' % options.index ) - seq_path = options.index + cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' ) + if not os.path.exists( cached_seqs_pointer_file ): + stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file ) + # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa, + # and the equCab2.fa file will contain fasta sequences. + seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file ) + if seq_path == '': + stop_err( 'No sequence data found for dbkey %s, so sequence data cannot be used.' % options.dbkey ) # Build command. diff --git a/tools/ngs_rna/cuffcompare_wrapper.xml b/tools/ngs_rna/cuffcompare_wrapper.xml index dc221c5d803..11e0ca75f0d 100644 --- a/tools/ngs_rna/cuffcompare_wrapper.xml +++ b/tools/ngs_rna/cuffcompare_wrapper.xml @@ -1,4 +1,4 @@ - + compare assembled transcripts to a reference annotation and track Cufflinks transcripts across multiple experiments @@ -17,12 +17,14 @@ ## Use sequence data? #if $seq_data.use_seq_data == "Yes": - -s + -s #if $seq_data.seq_source.index_source == "history": --ref_file=$seq_data.seq_source.ref_file #else: - --index=${seq_data.seq_source.index.fields.path} + --ref_file="None" #end if + --dbkey=${first_input.metadata.dbkey} + --index_dir=${GALAXY_DATA_INDEX_DIR} #end if ## Outputs. @@ -64,14 +66,7 @@ - - - - - - - - + diff --git a/tools/ngs_rna/cuffdiff_wrapper.py b/tools/ngs_rna/cuffdiff_wrapper.py index 000ac2c09df..4ba463df1b8 100644 --- a/tools/ngs_rna/cuffdiff_wrapper.py +++ b/tools/ngs_rna/cuffdiff_wrapper.py @@ -35,6 +35,20 @@ def stop_err( msg ): sys.stderr.write( "%s\n" % msg ) sys.exit() +# Copied from sam_to_bam.py: +def check_seq_file( dbkey, cached_seqs_pointer_file ): + seq_path = '' + for line in open( cached_seqs_pointer_file ): + line = line.rstrip( '\r\n' ) + if line and not line.startswith( '#' ) and line.startswith( 'index' ): + fields = line.split( '\t' ) + if len( fields ) < 3: + continue + if fields[1] == dbkey: + seq_path = fields[2].strip() + break + return seq_path + def __main__(): #Parse Command Line parser = optparse.OptionParser() @@ -69,7 +83,8 @@ def __main__(): # Bias correction options. parser.add_option( '-b', dest='do_bias_correction', action="store_true", help='Providing Cufflinks with a multifasta file via this option instructs it to run our new bias detection and correction algorithm which can significantly improve accuracy of transcript abundance estimates.') - parser.add_option( '', '--index', dest='index', help='The path of the reference genome' ) + parser.add_option( '', '--dbkey', dest='dbkey', help='The build of the reference dataset' ) + parser.add_option( '', '--index_dir', dest='index_dir', help='GALAXY_DATA_INDEX_DIR' ) parser.add_option( '', '--ref_file', dest='ref_file', help='The reference dataset from the history' ) # Outputs. @@ -108,16 +123,21 @@ def __main__(): # If doing bias correction, set/link to sequence file. if options.do_bias_correction: - if options.ref_file: + if options.ref_file != 'None': # Sequence data from history. # Create symbolic link to ref_file so that index will be created in working directory. seq_path = "ref.fa" os.symlink( options.ref_file, seq_path ) else: # Sequence data from loc file. - if not os.path.exists( options.index ): - stop_err( 'Reference genome %s not present, request it by reporting this error.' % options.index ) - seq_path = options.index + cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' ) + if not os.path.exists( cached_seqs_pointer_file ): + stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file ) + # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa, + # and the equCab2.fa file will contain fasta sequences. + seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file ) + if seq_path == '': + stop_err( 'No sequence data found for dbkey %s, so bias correction cannot be used.' % options.dbkey ) # Build command. diff --git a/tools/ngs_rna/cuffdiff_wrapper.xml b/tools/ngs_rna/cuffdiff_wrapper.xml index b4fe1bc6859..09e5efffa91 100644 --- a/tools/ngs_rna/cuffdiff_wrapper.xml +++ b/tools/ngs_rna/cuffdiff_wrapper.xml @@ -1,4 +1,4 @@ - + find significant changes in transcript expression, splicing, and promoter use @@ -42,12 +42,14 @@ ## Bias correction? #if $bias_correction.do_bias_correction == "Yes": - -b + -b #if $bias_correction.seq_source.index_source == "history": --ref_file=$bias_correction.seq_source.ref_file #else: - --index=${bias_correction.seq_source.index.fields.path} + --ref_file="None" #end if + --dbkey=${gtf_input.metadata.dbkey} + --index_dir=${GALAXY_DATA_INDEX_DIR} #end if ## Inputs. @@ -131,14 +133,7 @@ - - - - - - - - + diff --git a/tools/ngs_rna/cufflinks_wrapper.py b/tools/ngs_rna/cufflinks_wrapper.py index 683a8e8a7c8..d7051997bc9 100644 --- a/tools/ngs_rna/cufflinks_wrapper.py +++ b/tools/ngs_rna/cufflinks_wrapper.py @@ -10,6 +10,20 @@ def stop_err( msg ): sys.stderr.write( "%s\n" % msg ) sys.exit() +# Copied from sam_to_bam.py: +def check_seq_file( dbkey, cached_seqs_pointer_file ): + seq_path = '' + for line in open( cached_seqs_pointer_file ): + line = line.rstrip( '\r\n' ) + if line and not line.startswith( '#' ) and line.startswith( 'index' ): + fields = line.split( '\t' ) + if len( fields ) < 3: + continue + if fields[1] == dbkey: + seq_path = fields[2].strip() + break + return seq_path + def __main__(): #Parse Command Line parser = optparse.OptionParser() @@ -39,7 +53,8 @@ def __main__(): # Bias correction options. parser.add_option( '-b', dest='do_bias_correction', action="store_true", help='Providing Cufflinks with a multifasta file via this option instructs it to run our new bias detection and correction algorithm which can significantly improve accuracy of transcript abundance estimates.') - parser.add_option( '', '--index', dest='index', help='The path of the reference genome' ) + parser.add_option( '', '--dbkey', dest='dbkey', help='The build of the reference dataset' ) + parser.add_option( '', '--index_dir', dest='index_dir', help='GALAXY_DATA_INDEX_DIR' ) parser.add_option( '', '--ref_file', dest='ref_file', help='The reference dataset from the history' ) # Global model. @@ -68,16 +83,21 @@ def __main__(): # If doing bias correction, set/link to sequence file. if options.do_bias_correction: - if options.ref_file: + if options.ref_file != 'None': # Sequence data from history. # Create symbolic link to ref_file so that index will be created in working directory. seq_path = "ref.fa" os.symlink( options.ref_file, seq_path ) else: # Sequence data from loc file. - if not os.path.exists( options.index ): - stop_err( 'Reference genome %s not present, request it by reporting this error.' % options.index ) - seq_path = options.index + cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' ) + if not os.path.exists( cached_seqs_pointer_file ): + stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file ) + # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa, + # and the equCab2.fa file will contain fasta sequences. + seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file ) + if seq_path == '': + stop_err( 'No sequence data found for dbkey %s, so bias correction cannot be used.' % options.dbkey ) # Build command. diff --git a/tools/ngs_rna/cufflinks_wrapper.xml b/tools/ngs_rna/cufflinks_wrapper.xml index 738e02b4ee0..6f002c6626e 100644 --- a/tools/ngs_rna/cufflinks_wrapper.xml +++ b/tools/ngs_rna/cufflinks_wrapper.xml @@ -1,4 +1,4 @@ - + transcript assembly and FPKM (RPKM) estimates for RNA-Seq data @@ -29,12 +29,14 @@ ## Bias correction? #if $bias_correction.do_bias_correction == "Yes": - -b + -b #if $bias_correction.seq_source.index_source == "history": --ref_file=$bias_correction.seq_source.ref_file #else: - --index=${bias_correction.seq_source.index.fields.path} + --ref_file="None" #end if + --dbkey=${input.metadata.dbkey} + --index_dir=${GALAXY_DATA_INDEX_DIR} #end if ## Multi-read correct? @@ -65,15 +67,15 @@ - - + + - + - + @@ -81,14 +83,7 @@ - - - - - - - - + diff --git a/tools/ngs_rna/cuffmerge_wrapper.py b/tools/ngs_rna/cuffmerge_wrapper.py index ae2c6cef849..6d7fc39d08d 100644 --- a/tools/ngs_rna/cuffmerge_wrapper.py +++ b/tools/ngs_rna/cuffmerge_wrapper.py @@ -8,6 +8,20 @@ def stop_err( msg ): sys.stderr.write( '%s\n' % msg ) sys.exit() +# Copied from sam_to_bam.py: +def check_seq_file( dbkey, cached_seqs_pointer_file ): + seq_path = '' + for line in open( cached_seqs_pointer_file ): + line = line.rstrip( '\r\n' ) + if line and not line.startswith( '#' ) and line.startswith( 'index' ): + fields = line.split( '\t' ) + if len( fields ) < 3: + continue + if fields[1] == dbkey: + seq_path = fields[2].strip() + break + return seq_path + def __main__(): #Parse Command Line parser = optparse.OptionParser() @@ -17,7 +31,8 @@ def __main__(): # Wrapper / Galaxy options. - parser.add_option( '', '--index', dest='index', help='The path of the reference genome' ) + parser.add_option( '', '--dbkey', dest='dbkey', help='The build of the reference dataset' ) + parser.add_option( '', '--index_dir', dest='index_dir', help='GALAXY_DATA_INDEX_DIR' ) parser.add_option( '', '--ref_file', dest='ref_file', help='The reference dataset from the history' ) # Outputs. @@ -46,16 +61,21 @@ def __main__(): # Set/link to sequence file. if options.use_seq_data: - if options.ref_file: + if options.ref_file != 'None': # Sequence data from history. # Create symbolic link to ref_file so that index will be created in working directory. seq_path = "ref.fa" os.symlink( options.ref_file, seq_path ) else: # Sequence data from loc file. - if not os.path.exists( options.index ): - stop_err( 'Reference genome %s not present, request it by reporting this error.' % options.index ) - seq_path = options.index + cached_seqs_pointer_file = os.path.join( options.index_dir, 'sam_fa_indices.loc' ) + if not os.path.exists( cached_seqs_pointer_file ): + stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file ) + # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa, + # and the equCab2.fa file will contain fasta sequences. + seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file ) + if seq_path == '': + stop_err( 'No sequence data found for dbkey %s, so sequence data cannot be used.' % options.dbkey ) # Build command. diff --git a/tools/ngs_rna/cuffmerge_wrapper.xml b/tools/ngs_rna/cuffmerge_wrapper.xml index 0209de52308..09a687b468e 100644 --- a/tools/ngs_rna/cuffmerge_wrapper.xml +++ b/tools/ngs_rna/cuffmerge_wrapper.xml @@ -1,4 +1,4 @@ - + merge together several Cufflinks assemblies @@ -16,12 +16,14 @@ ## Use sequence data? #if $seq_data.use_seq_data == "Yes": - -s + -s #if $seq_data.seq_source.index_source == "history": --ref_file=$seq_data.seq_source.ref_file #else: - --index=${seq_data.seq_source.index.fields.path} + --ref_file="None" #end if + --dbkey=${first_input.metadata.dbkey} + --index_dir=${GALAXY_DATA_INDEX_DIR} #end if ## Outputs. @@ -62,14 +64,7 @@ - - - - - - - - + From caa16ac59e05228083f6372478f4863c65829469 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 30 Jul 2013 12:54:02 -0400 Subject: [PATCH 318/318] See previous commit message. --- tools/samtools/sam_pileup.py | 25 ++++++++++++++++---- tools/samtools/sam_pileup.xml | 15 ++++-------- tools/samtools/sam_to_bam.py | 36 +++++++++++++++++++++++++---- tools/samtools/sam_to_bam.xml | 21 +++++++---------- tools/samtools/samtools_mpileup.xml | 18 +++++++-------- 5 files changed, 74 insertions(+), 41 deletions(-) diff --git a/tools/samtools/sam_pileup.py b/tools/samtools/sam_pileup.py index b236c95ed84..50168fc2fe6 100644 --- a/tools/samtools/sam_pileup.py +++ b/tools/samtools/sam_pileup.py @@ -8,7 +8,8 @@ usage: %prog [options] -o, --output1=o: Output pileup -R, --ref=R: Reference file type -n, --ownFile=n: User-supplied fasta reference file - -g, --index=g: Path of the indexed reference genome + -d, --dbkey=d: dbkey of user-supplied file + -x, --indexDir=x: Index directory -b, --bamIndex=b: BAM index file -s, --lastCol=s: Print the mapping quality as the last column -i, --indels=i: Only output lines containing indels @@ -30,9 +31,24 @@ def stop_err( msg ): sys.stderr.write( '%s\n' % msg ) sys.exit() +def check_seq_file( dbkey, GALAXY_DATA_INDEX_DIR ): + seqFile = '%s/sam_fa_indices.loc' % GALAXY_DATA_INDEX_DIR + seqPath = '' + for line in open( seqFile ): + line = line.rstrip( '\r\n' ) + if line and not line.startswith( '#' ) and line.startswith( 'index' ): + fields = line.split( '\t' ) + if len( fields ) < 3: + continue + if fields[1] == dbkey: + seqPath = fields[2].strip() + break + return seqPath + def __main__(): #Parse Command Line options, args = doc_optparse.parse( __doc__ ) + seqPath = check_seq_file( options.dbkey, options.indexDir ) # output version # of tool try: tmp = tempfile.NamedTemporaryFile().name @@ -61,6 +77,7 @@ def __main__(): tmpf1 = tempfile.NamedTemporaryFile( dir=tmpDir ) tmpf1_name = tmpf1.name tmpf1.close() + tmpf1fai_name = '%s.fai' % tmpf1_name #link bam and bam index to working directory (can't move because need to leave original) os.symlink( options.input1, tmpf0bam_name ) os.symlink( options.bamIndex, tmpf0bambai_name ) @@ -83,9 +100,9 @@ def __main__(): try: #index reference if necessary and prepare pileup command if options.ref == 'indexed': - if not os.path.exists( "%s.fai" % options.index ): - raise Exception, "Indexed genome %s not present, request it by reporting this error." % options.index - cmd = cmd % ( opts, options.index, tmpf0bam_name, options.output1 ) + if not os.path.exists( "%s.fai" % seqPath ): + raise Exception, "No sequences are available for '%s', request them by reporting this error." % options.dbkey + cmd = cmd % ( opts, seqPath, tmpf0bam_name, options.output1 ) elif options.ref == 'history': os.symlink( options.ownFile, tmpf1_name ) cmdIndex = 'samtools faidx %s' % ( tmpf1_name ) diff --git a/tools/samtools/sam_pileup.xml b/tools/samtools/sam_pileup.xml index 37897b395cc..e51377f5aac 100644 --- a/tools/samtools/sam_pileup.xml +++ b/tools/samtools/sam_pileup.xml @@ -1,4 +1,4 @@ - + from BAM dataset samtools @@ -11,8 +11,10 @@ #if $refOrHistory.reference == "history": --ownFile=$refOrHistory.ownFile #else: - --index=${refOrHistory.index.fields.path} + --ownFile="None" #end if + --dbkey=${input1.metadata.dbkey} + --indexDir=${GALAXY_DATA_INDEX_DIR} --bamIndex=${input1.metadata.bam_index} --lastCol=$lastCol --indels=$indels @@ -39,13 +41,7 @@ - - - - - - - + @@ -104,7 +100,6 @@ --> - diff --git a/tools/samtools/sam_to_bam.py b/tools/samtools/sam_to_bam.py index 4a6b8a19bda..3db373b9401 100644 --- a/tools/samtools/sam_to_bam.py +++ b/tools/samtools/sam_to_bam.py @@ -3,24 +3,43 @@ Converts SAM data to sorted BAM data. usage: sam_to_bam.py [options] --input1: SAM file to be converted - --index: path of the indexed reference genome + --dbkey: dbkey value --ref_file: Reference file if choosing from history --output1: output dataset in bam format + --index_dir: GALAXY_DATA_INDEX_DIR """ -import optparse, os, sys, subprocess, tempfile, shutil +import optparse, os, sys, subprocess, tempfile, shutil, gzip +from galaxy import eggs +import pkg_resources; pkg_resources.require( "bx-python" ) +from bx.cookbook import doc_optparse +from galaxy import util def stop_err( msg ): sys.stderr.write( '%s\n' % msg ) sys.exit() +def check_seq_file( dbkey, cached_seqs_pointer_file ): + seq_path = '' + for line in open( cached_seqs_pointer_file ): + line = line.rstrip( '\r\n' ) + if line and not line.startswith( '#' ) and line.startswith( 'index' ): + fields = line.split( '\t' ) + if len( fields ) < 3: + continue + if fields[1] == dbkey: + seq_path = fields[2].strip() + break + return seq_path + def __main__(): #Parse Command Line parser = optparse.OptionParser() parser.add_option( '', '--input1', dest='input1', help='The input SAM dataset' ) - parser.add_option( '', '--index', dest='index', help='The path of the indexed reference genome' ) + parser.add_option( '', '--dbkey', dest='dbkey', help='The build of the reference dataset' ) parser.add_option( '', '--ref_file', dest='ref_file', help='The reference dataset from the history' ) parser.add_option( '', '--output1', dest='output1', help='The output BAM dataset' ) + parser.add_option( '', '--index_dir', dest='index_dir', help='GALAXY_DATA_INDEX_DIR' ) ( options, args ) = parser.parse_args() # output version # of tool @@ -42,17 +61,24 @@ def __main__(): except: sys.stdout.write( 'Could not determine Samtools version\n' ) + cached_seqs_pointer_file = '%s/sam_fa_indices.loc' % options.index_dir + if not os.path.exists( cached_seqs_pointer_file ): + stop_err( 'The required file (%s) does not exist.' % cached_seqs_pointer_file ) + # If found for the dbkey, seq_path will look something like /galaxy/data/equCab2/sam_index/equCab2.fa, + # and the equCab2.fa file will contain fasta sequences. + seq_path = check_seq_file( options.dbkey, cached_seqs_pointer_file ) tmp_dir = tempfile.mkdtemp( dir='.' ) if not options.ref_file or options.ref_file == 'None': # We're using locally cached reference sequences( e.g., /galaxy/data/equCab2/sam_index/equCab2.fa ). # The indexes for /galaxy/data/equCab2/sam_index/equCab2.fa will be contained in # a file named /galaxy/data/equCab2/sam_index/equCab2.fa.fai - fai_index_file_path = '%s.fai' % options.index + fai_index_file_base = seq_path + fai_index_file_path = '%s.fai' % seq_path if not os.path.exists( fai_index_file_path ): #clean up temp files if os.path.exists( tmp_dir ): shutil.rmtree( tmp_dir ) - stop_err( 'Indexed genome %s not present, request it by reporting this error.' % options.index ) + stop_err( 'No sequences are available for build (%s), request them by reporting this error.' % options.dbkey ) else: try: # Create indexes for history reference ( e.g., ~/database/files/000/dataset_1.dat ) using samtools faidx, which will: diff --git a/tools/samtools/sam_to_bam.xml b/tools/samtools/sam_to_bam.xml index 03a0c478421..7364bb4fdac 100644 --- a/tools/samtools/sam_to_bam.xml +++ b/tools/samtools/sam_to_bam.xml @@ -1,4 +1,4 @@ - + converts SAM format to BAM format samtools @@ -7,11 +7,13 @@ sam_to_bam.py --input1=$source.input1 #if $source.index_source == "history": + --dbkey=${ref_file.metadata.dbkey} --ref_file=$source.ref_file #else - --index=${source.index.fields.path} + --dbkey=${input1.metadata.dbkey} #end if --output1=$output1 + --index_dir=${GALAXY_DATA_INDEX_DIR} @@ -20,19 +22,13 @@ - - - - - - - - - + + + - + @@ -80,7 +76,6 @@ --> - diff --git a/tools/samtools/samtools_mpileup.xml b/tools/samtools/samtools_mpileup.xml index 81a7f3dbf2f..a8280630e77 100644 --- a/tools/samtools/samtools_mpileup.xml +++ b/tools/samtools/samtools_mpileup.xml @@ -1,4 +1,4 @@ - + SNP and indel caller samtools @@ -59,22 +59,22 @@ - - - - + + + + - + - - - + + +