diff --git a/README.rst b/README.rst index 66c10038352..54e5929e0b2 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ Not all dependencies are included for the tools provided in the sample https://wiki.galaxyproject.org/Admin/Tools/ToolDependencies Issues ------- +====== Issues can be submitted to trello via the `galaxyproject website `__ and viewed on the `Galaxy @@ -51,6 +51,19 @@ Galaxy Development Galaxy welcomes new development! There is extensive documentation on developing with Galaxy on the `wiki `__. +Source Repository +----------------- + +Galaxy development has moved to `Github +`__. + +**Developers** should develop against the ``dev`` branch in the Github +repository. + +**Deployers** can continue to pull Galaxy changes from Bitbucket without +interruption. Alternatively, deployers can track the ``master`` branch in +Github. + Syncing a Fork -------------- diff --git a/client/galaxy/scripts/mvc/tools/tools-form-base.js b/client/galaxy/scripts/mvc/tools/tools-form-base.js index 94a0bfece5f..0da0f7a1a32 100644 --- a/client/galaxy/scripts/mvc/tools/tools-form-base.js +++ b/client/galaxy/scripts/mvc/tools/tools-form-base.js @@ -108,7 +108,7 @@ define(['utils/utils', 'utils/deferred', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc', var new_state = self.tree.finalize(); if (!_.isEqual(new_state, current_state)) { current_state = new_state; - self._updateModel($.extend({}, current_state)); + self._updateModel($.extend(true, {}, current_state)); } }); }); diff --git a/client/galaxy/scripts/templates/tool_search.handlebars b/client/galaxy/scripts/templates/tool_search.handlebars index bd035f2e92b..995b9ce4af4 100644 --- a/client/galaxy/scripts/templates/tool_search.handlebars +++ b/client/galaxy/scripts/templates/tool_search.handlebars @@ -1,3 +1,3 @@ - + - \ No newline at end of file + diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 51a75abd7d4..65bfb9ebdac 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -180,7 +180,7 @@ - + diff --git a/config/tool_data_table_conf.xml.sample b/config/tool_data_table_conf.xml.sample index 910889e8320..82b3079f07c 100644 --- a/config/tool_data_table_conf.xml.sample +++ b/config/tool_data_table_conf.xml.sample @@ -65,4 +65,9 @@ value, path + + + name, url, value + +
diff --git a/display_applications/igv/bam.xml b/display_applications/igv/bam.xml index 1e7058e01c9..275e6aa16ef 100644 --- a/display_applications/igv/bam.xml +++ b/display_applications/igv/bam.xml @@ -93,7 +93,20 @@ #end if - + + + + + + ${ $dataset.dbkey == $value } + + + http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${bam_file.qp}&genome=${bam_file.dbkey}&merge=true&name=${qp( $bam_file.name )} + + + + + diff --git a/display_applications/igv/vcf.xml b/display_applications/igv/vcf.xml index a9276745b0c..00321d22e77 100644 --- a/display_applications/igv/vcf.xml +++ b/display_applications/igv/vcf.xml @@ -93,7 +93,20 @@ #end if - + + + + + + ${ $dataset.dbkey == $value } + + + http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${bgzip_file.qp}&genome=$bgzip_file.dbkey&merge=true&name=${qp( $bgzip_file.name )} + + + + + diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index f6cc5f919b9..ffad22f6587 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -88,7 +88,7 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ): # Load proprietary datatype converters and display applications. self.installed_repository_manager.load_proprietary_converters_and_display_applications() # Load datatype display applications defined in local datatypes_conf.xml - self.datatypes_registry.load_display_applications() + self.datatypes_registry.load_display_applications( self ) # Load datatype converters defined in local datatypes_conf.xml self.datatypes_registry.load_datatype_converters( self.toolbox ) # Load external metadata tool diff --git a/lib/galaxy/datatypes/display_applications/application.py b/lib/galaxy/datatypes/display_applications/application.py index 74c20e139e2..75ffcf6368f 100644 --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -80,21 +80,57 @@ class DisplayApplicationLink( object ): return True class DynamicDisplayApplicationBuilder( object ): - @classmethod + def __init__( self, elem, display_application, build_sites ): - rval = [] filename = None + data_table = None if elem.get( 'site_type', None ) is not None: filename = build_sites.get( elem.get( 'site_type' ) ) else: filename = elem.get( 'from_file', None ) - assert filename is not None, 'Filename and id attributes required for dynamic_links' + if filename is None: + data_table_name = elem.get( 'from_data_table', None ) + if data_table_name: + data_table = display_application.app.tool_data_tables.get( data_table_name, None ) + assert data_table is not None, 'Unable to find data table named "%s".' % data_table_name + + assert filename is not None or data_table is not None,'Filename or data Table is required for dynamic_links.' skip_startswith = elem.get( 'skip_startswith', None ) separator = elem.get( 'separator', '\t' ) - id_col = int( elem.get( 'id', None ) ) - name_col = int( elem.get( 'name', id_col ) ) - dynamic_params = {} + id_col = elem.get( 'id', None ) + try: + id_col = int( id_col ) + except: + if data_table: + if id_col is None: + id_col = data_table.columns.get( 'id', None ) + if id_col is None: + id_col = data_table.columns.get( 'value', None ) + try: + id_col = int( id_col ) + except: + # id is set to a string or None, use column by that name if available + id_col = data_table.columns.get( id_col, None ) + id_col = int( id_col ) + name_col = elem.get( 'name', None ) + try: + name_col = int( name_col ) + except: + if data_table: + if name_col is None: + name_col = data_table.columns.get( 'name', None ) + else: + name_col = data_table.columns.get( name_col, None ) + else: + name_col = None + if name_col is None: + name_col = id_col max_col = max( id_col, name_col ) + dynamic_params = {} + if data_table is not None: + max_col = max( [ max_col ] + data_table.columns.values() ) + for key, value in data_table.columns.items(): + dynamic_params[key] = { 'column': value, 'split': False, 'separator': ',' } for dynamic_param in elem.findall( 'dynamic_param' ): name = dynamic_param.get( 'name' ) value = int( dynamic_param.get( 'value' ) ) @@ -102,27 +138,38 @@ class DynamicDisplayApplicationBuilder( object ): param_separator = dynamic_param.get( 'separator', ',' ) max_col = max( max_col, value ) dynamic_params[name] = { 'column': value, 'split': split, 'separator': param_separator } - for line in open( filename ): - if not skip_startswith or not line.startswith( skip_startswith ): - line = line.rstrip( '\n\r' ) - if not line: - continue - fields = line.split( separator ) - if len( fields ) > max_col: - new_elem = deepcopy( elem ) - new_elem.set( 'id', fields[id_col] ) - new_elem.set( 'name', fields[name_col] ) - dynamic_values = {} - for key, attributes in dynamic_params.iteritems(): - value = fields[ attributes[ 'column' ] ] - if attributes['split']: - value = value.split( attributes['separator'] ) - dynamic_values[key] = value - #now populate - rval.append( DisplayApplicationLink.from_elem( new_elem, display_application, other_values = dynamic_values ) ) + if filename: + data_iter = open( filename ) + elif data_table: + version, data_iter = data_table.get_version_fields() + display_application.add_data_table_watch( data_table.name, version ) + links = [] + for line in data_iter: + if isinstance( line, basestring ): + if not skip_startswith or not line.startswith( skip_startswith ): + line = line.rstrip( '\n\r' ) + if not line: + continue + fields = line.split( separator ) else: - log.warning( 'Invalid dynamic display application link specified in %s: "%s"' % ( filename, line ) ) - self.links = rval + continue + else: + fields = line + if len( fields ) > max_col: + new_elem = deepcopy( elem ) + new_elem.set( 'id', fields[id_col] ) + new_elem.set( 'name', fields[name_col] ) + dynamic_values = {} + for key, attributes in dynamic_params.iteritems(): + value = fields[ attributes[ 'column' ] ] + if attributes['split']: + value = value.split( attributes['separator'] ) + dynamic_values[key] = value + #now populate + links.append( DisplayApplicationLink.from_elem( new_elem, display_application, other_values = dynamic_values ) ) + else: + log.warning( 'Invalid dynamic display application link specified in %s: "%s"' % ( filename, line ) ) + self.links = links def __iter__( self ): return iter( self.links ) @@ -166,12 +213,12 @@ class PopulatedDisplayApplicationLink( object ): class DisplayApplication( object ): @classmethod - def from_file( cls, filename, datatypes_registry ): - return cls.from_elem( parse_xml( filename ).getroot(), datatypes_registry, filename=filename ) + def from_file( cls, filename, app ): + return cls.from_elem( parse_xml( filename ).getroot(), app, filename=filename ) @classmethod - def from_elem( cls, elem, datatypes_registry, filename=None ): + def from_elem( cls, elem, app, filename=None ): att_dict = cls._get_attributes_from_elem( elem ) - rval = DisplayApplication( att_dict['id'], att_dict['name'], datatypes_registry, att_dict['version'], filename=filename, elem=elem ) + rval = DisplayApplication( att_dict['id'], att_dict['name'], app, att_dict['version'], filename=filename, elem=elem ) rval._load_links_from_elem( elem ) return rval @classmethod @@ -181,29 +228,32 @@ class DisplayApplication( object ): name = elem.get( 'name', display_id ) version = elem.get( 'version', None ) return dict( id=display_id, name=name, version=version ) - def __init__( self, display_id, name, datatypes_registry, version = None, filename=None, elem=None ): + def __init__( self, display_id, name, app, version = None, filename=None, elem=None ): self.id = display_id self.name = name - self.datatypes_registry = datatypes_registry + self.app = app if version is None: version = "1.0.0" self.version = version self.links = odict() self._filename = filename self._elem = elem + self._data_table_versions = {} def _load_links_from_elem( self, elem ): for link_elem in elem.findall( 'link' ): link = DisplayApplicationLink.from_elem( link_elem, self ) if link: self.links[ link.id ] = link for dynamic_links in elem.findall( 'dynamic_links' ): - for link in DynamicDisplayApplicationBuilder( dynamic_links, self, self.datatypes_registry.build_sites ): + for link in DynamicDisplayApplicationBuilder( dynamic_links, self, self.app.datatypes_registry.build_sites ): self.links[ link.id ] = link def get_link( self, link_name, data, dataset_hash, user_hash, trans, app_kwds ): #returns a link object with data knowledge to generate links + self._check_and_reload() return PopulatedDisplayApplicationLink( self.links[ link_name ], data, dataset_hash, user_hash, trans, app_kwds ) def filter_by_dataset( self, data, trans ): - filtered = DisplayApplication( self.id, self.name, self.datatypes_registry, version = self.version ) + self._check_and_reload() + filtered = DisplayApplication( self.id, self.name, self.app, version = self.version ) for link_name, link_value in self.links.iteritems(): if link_value.filter_by_dataset( data, trans ): filtered.links[link_name] = link_value @@ -222,9 +272,24 @@ class DisplayApplication( object ): # clear old links for key in self.links.keys(): del self.links[key] + #clear data table versions: + for key in self._data_table_versions.keys(): + del self._data_table_versions[ key ] # Set new attributes for key, value in attr_dict.iteritems(): setattr( self, key, value ) # Load new links self._load_links_from_elem( elem ) return self + def add_data_table_watch( self, table_name, version=None ): + self._data_table_versions[ table_name ] = version + def _requires_reload( self ): + for key, value in self._data_table_versions.iteritems(): + table = self.app.tool_data_tables.get( key, None ) + if table and not table.is_current_version( value ): + return True + return False + def _check_and_reload( self ): + if self._requires_reload(): + self.reload() + diff --git a/lib/galaxy/datatypes/display_applications/parameters.py b/lib/galaxy/datatypes/display_applications/parameters.py index b4d382a626c..8105fcac217 100644 --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -59,7 +59,7 @@ class DisplayApplicationDataParameter( DisplayApplicationParameter ): @property def formats( self ): if self.extensions: - return tuple( map( type, map( self.link.display_application.datatypes_registry.get_datatype_by_extension, self.extensions ) ) ) + return tuple( map( type, map( self.link.display_application.app.datatypes_registry.get_datatype_by_extension, self.extensions ) ) ) return None def _get_dataset_like_object( self, other_values ): #this returned object has file_name, state, and states attributes equivalent to a DatasetAssociation diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 76a817a5d00..be9704015d7 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -546,7 +546,7 @@ class Registry( object ): else: self.log.exception( "Error loading converter (%s): %s" % ( converter_path, str( e ) ) ) - def load_display_applications( self, installed_repository_dict=None, deactivate=False ): + def load_display_applications( self, app, installed_repository_dict=None, deactivate=False ): """ If deactivate is False, add display applications from self.display_app_containers or self.proprietary_display_app_containers to appropriate datatypes. If deactivate is @@ -570,7 +570,7 @@ class Registry( object ): config_path = os.path.join( self.display_applications_path, display_file ) try: inherit = galaxy.util.string_as_bool( display_app.get( 'inherit', 'False' ) ) - display_app = DisplayApplication.from_file( config_path, self ) + display_app = DisplayApplication.from_file( config_path, app ) if display_app: if display_app.id in self.display_applications: if deactivate: diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 98530928512..c02fe5afb8d 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -203,6 +203,94 @@ class Sequence( data.Text ): return None raise NotImplementedError("Can't split generic sequence files") + def get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count): + """ + Uses a Table of Contents dict, parsed from an FQTOC file, to come up with a set of + shell commands that will extract the parts necessary + >>> three_sections=[dict(start=0, end=74, sequences=10), dict(start=74, end=148, sequences=10), dict(start=148, end=148+76, sequences=10)] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=0, sequence_count=10) + ['dd bs=1 skip=0 count=74 if=./input.gz 2> /dev/null >> ./output.gz'] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=1, sequence_count=5) + ['(dd bs=1 skip=0 count=74 if=./input.gz 2> /dev/null )| zcat | ( tail -n +5 2> /dev/null) | head -20 | gzip -c >> ./output.gz'] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=0, sequence_count=20) + ['dd bs=1 skip=0 count=148 if=./input.gz 2> /dev/null >> ./output.gz'] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=5, sequence_count=10) + ['(dd bs=1 skip=0 count=74 if=./input.gz 2> /dev/null )| zcat | ( tail -n +21 2> /dev/null) | head -20 | gzip -c >> ./output.gz', '(dd bs=1 skip=74 count=74 if=./input.gz 2> /dev/null )| zcat | ( tail -n +1 2> /dev/null) | head -20 | gzip -c >> ./output.gz'] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=10, sequence_count=10) + ['dd bs=1 skip=74 count=74 if=./input.gz 2> /dev/null >> ./output.gz'] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=5, sequence_count=20) + ['(dd bs=1 skip=0 count=74 if=./input.gz 2> /dev/null )| zcat | ( tail -n +21 2> /dev/null) | head -20 | gzip -c >> ./output.gz', 'dd bs=1 skip=74 count=74 if=./input.gz 2> /dev/null >> ./output.gz', '(dd bs=1 skip=148 count=76 if=./input.gz 2> /dev/null )| zcat | ( tail -n +1 2> /dev/null) | head -20 | gzip -c >> ./output.gz'] + """ + sections = toc_file['sections'] + result = [] + + current_sequence = long(0) + i=0 + # skip to the section that contains my starting sequence + while i < len(sections) and start_sequence >= current_sequence + long(sections[i]['sequences']): + current_sequence += long(sections[i]['sequences']) + i += 1 + if i == len(sections): # bad input data! + raise Exception('No FQTOC section contains starting sequence %s' % start_sequence) + + # These two variables act as an accumulator for consecutive entire blocks that + # can be copied verbatim (without decompressing) + start_chunk = long(-1) + end_chunk = long(-1) + copy_chunk_cmd = 'dd bs=1 skip=%s count=%s if=%s 2> /dev/null >> %s' + + while sequence_count > 0 and i < len(sections): + # we need to extract partial data. So, find the byte offsets of the chunks that contain the data we need + # use a combination of dd (to pull just the right sections out) tail (to skip lines) and head (to get the + # right number of lines + sequences = long(sections[i]['sequences']) + skip_sequences = start_sequence-current_sequence + sequences_to_extract = min(sequence_count, sequences-skip_sequences) + start_copy = long(sections[i]['start']) + end_copy = long(sections[i]['end']) + if sequences_to_extract < sequences: + if start_chunk > -1: + result.append(copy_chunk_cmd % (start_chunk, end_chunk-start_chunk, input_name, output_name)) + start_chunk = -1 + # extract, unzip, trim, recompress + result.append('(dd bs=1 skip=%s count=%s if=%s 2> /dev/null )| zcat | ( tail -n +%s 2> /dev/null) | head -%s | gzip -c >> %s' % + (start_copy, end_copy-start_copy, input_name, skip_sequences*4+1, sequences_to_extract*4, output_name)) + else: # whole section - add it to the start_chunk/end_chunk accumulator + if start_chunk == -1: + start_chunk = start_copy + end_chunk = end_copy + sequence_count -= sequences_to_extract + start_sequence += sequences_to_extract + current_sequence += sequences + i += 1 + if start_chunk > -1: + result.append(copy_chunk_cmd % (start_chunk, end_chunk-start_chunk, input_name, output_name)) + + if sequence_count > 0: + raise Exception('%s sequences not found in file' % sequence_count) + + return result + get_split_commands_with_toc = staticmethod(get_split_commands_with_toc) + + def get_split_commands_sequential(is_compressed, input_name, output_name, start_sequence, sequence_count): + """ + Does a brain-dead sequential scan & extract of certain sequences + >>> Sequence.get_split_commands_sequential(True, './input.gz', './output.gz', start_sequence=0, sequence_count=10) + ['zcat "./input.gz" | ( tail -n +1 2> /dev/null) | head -40 | gzip -c > "./output.gz"'] + >>> Sequence.get_split_commands_sequential(False, './input.fastq', './output.fastq', start_sequence=10, sequence_count=10) + ['tail -n +41 "./input.fastq" 2> /dev/null | head -40 > "./output.fastq"'] + """ + start_line = start_sequence * 4 + line_count = sequence_count * 4 + # TODO: verify that tail can handle 64-bit numbers + if is_compressed: + cmd = 'zcat "%s" | ( tail -n +%s 2> /dev/null) | head -%s | gzip -c' % (input_name, start_line+1, line_count) + else: + cmd = 'tail -n +%s "%s" 2> /dev/null | head -%s' % (start_line+1, input_name, line_count) + cmd += ' > "%s"' % output_name + + return [cmd] + get_split_commands_sequential = staticmethod(get_split_commands_sequential) class Alignment( data.Text ): """Class describing an alignment""" diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 9e469ba9659..bbd17ffd1a6 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -1,6 +1,5 @@ import os import re -import shutil import string import tarfile import tempfile @@ -23,6 +22,7 @@ from .panel import ToolPanelElements from .panel import ToolSectionLabel from .panel import ToolSection from .panel import panel_item_types +from .integrated_panel import ManagesIntegratedToolPanelMixin from .lineages import LineageMap from .tags import tool_tag_manager @@ -37,22 +37,7 @@ import logging log = logging.getLogger( __name__ ) -INTEGRATED_TOOL_PANEL_DESCRIPTION = """ -This is Galaxy's integrated tool panel and should be modified directly only for -reordering tools inside a section. Each time Galaxy starts up, this file is -synchronized with the various tool config files: tools, sections and labels -added to one of these files, will be added also here in the appropriate place, -while elements removed from the tool config files will be correspondingly -deleted from this file. -To modify locally managed tools (e.g. from tool_conf.xml) modify that file -directly and restart Galaxy. Whenever possible Tool Shed managed tools (e.g. -from shed_tool_conf.xml) should be managed from within the Galaxy interface or -via its API - but if changes are necessary (such as to hide a tool or re-assign -its section) modify that file and restart Galaxy. -""" - - -class AbstractToolBox( object, Dictifiable ): +class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): """ Abstract container for managing a ToolPanel - containing tools and workflows optionally in labelled sections. @@ -78,15 +63,8 @@ class AbstractToolBox( object, Dictifiable ): self._index = 0 self.data_manager_tools = odict() self._lineage_map = LineageMap( app ) - # File that contains the XML section and tool tags from all tool panel config files integrated into a - # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator - # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. - self._integrated_tool_panel_config = app.config.integrated_tool_panel_config - # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. - self._integrated_tool_panel = ToolPanelElements() - self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 - if self._integrated_tool_panel_config_has_contents: - self._load_integrated_tool_panel_keys() + # Sets self._integrated_tool_panel and self._integrated_tool_panel_config_has_contents + self._init_integrated_tool_panel( app.config ) # The following refers to the tool_path config setting for backward compatibility. The shed-related # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the tag. self._tool_root_dir = tool_root_dir @@ -98,12 +76,7 @@ class AbstractToolBox( object, Dictifiable ): if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents: # Load self._tool_panel based on the order in self._integrated_tool_panel. self._load_tool_panel() - if app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() + self._save_integrated_tool_panel() def create_tool( self, config_file, repository_id=None, guid=None, **kwds ): raise NotImplementedError() @@ -215,9 +188,8 @@ class AbstractToolBox( object, Dictifiable ): for index, my_shed_tool_conf in enumerate( self._dynamic_tool_confs ): if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: self._dynamic_tool_confs[ index ] = shed_conf - if integrated_panel_changes and app.config.update_integrated_tool_panel: - # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - self._write_integrated_tool_panel_config_file() + if integrated_panel_changes: + self._save_integrated_tool_panel() app.reindex_tool_search() def get_section( self, section_id, new_label=None, create_if_needed=False ): @@ -237,7 +209,7 @@ class AbstractToolBox( object, Dictifiable ): 'version': '', } tool_section = ToolSection( section_dict ) - self._tool_panel[ tool_panel_section_key ] = tool_section + self._tool_panel.append_section( tool_panel_section_key, tool_section ) log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) else: tool_section = None @@ -400,52 +372,6 @@ class AbstractToolBox( object, Dictifiable ): elif elem.tag == 'label': self._integrated_tool_panel.stub_label( key ) - def _write_integrated_tool_panel_config_file( self ): - """ - Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators - use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. - """ - fd, filename = tempfile.mkstemp() - os.write( fd, '\n' ) - os.write( fd, '\n' ) - os.write( fd, ' \n') - for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): - if item: - if item_type == panel_item_types.TOOL: - os.write( fd, ' \n' % item.id ) - elif item_type == panel_item_types.WORKFLOW: - os.write( fd, ' \n' % item.id ) - elif item_type == panel_item_types.LABEL: - label_id = item.id or '' - label_text = item.text or '' - label_version = item.version or '' - os.write( fd, ' \n' ) - os.close( fd ) - shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) - os.chmod( self._integrated_tool_panel_config, 0644 ) - def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" if tool_version: @@ -457,9 +383,9 @@ class AbstractToolBox( object, Dictifiable ): if tool_id in self._tools_by_id and not get_all_versions: if tool_version and tool_version in self._tool_versions_by_id[ tool_id ]: return self._tool_versions_by_id[ tool_id ][ tool_version ] - #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) + # tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) return self._tools_by_id[ tool_id ] - #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions + # exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: @@ -469,7 +395,7 @@ class AbstractToolBox( object, Dictifiable ): if lineage_tool: rval.append( lineage_tool ) if not rval: - #still no tool, do a deeper search and try to match by old ids + # still no tool, do a deeper search and try to match by old ids for tool in self._tools_by_id.itervalues(): if tool.old_id == tool_id: rval.append( tool ) @@ -478,14 +404,14 @@ class AbstractToolBox( object, Dictifiable ): return rval else: if tool_version: - #return first tool with matching version + # return first tool with matching version for tool in rval: if tool.version == tool_version: return tool - #No tool matches by version, simply return the first available tool found + # No tool matches by version, simply return the first available tool found return rval[0] - #We now likely have a Toolshed guid passed in, but no supporting database entries - #If the tool exists by exact id and is loaded then provide exact match within a list + # We now likely have a Toolshed guid passed in, but no supporting database entries + # If the tool exists by exact id and is loaded then provide exact match within a list if tool_id in self._tools_by_id: return[ self._tools_by_id[ tool_id ] ] return None @@ -777,13 +703,7 @@ class AbstractToolBox( object, Dictifiable ): if async: self._load_tool_panel() - - if self.app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() + self._save_integrated_tool_panel() return tool.id except Exception: log.exception("Failed to load potential tool %s." % tool_file) @@ -1012,7 +932,7 @@ class AbstractToolBox( object, Dictifiable ): break if tool_id in self.data_manager_tools: del self.data_manager_tools[ tool_id ] - #TODO: do we need to manually remove from the integrated panel here? + # TODO: do we need to manually remove from the integrated panel here? message = "Removed the tool:
" message += "name: %s
" % tool.name message += "id: %s
" % tool.id diff --git a/lib/galaxy/tools/toolbox/integrated_panel.py b/lib/galaxy/tools/toolbox/integrated_panel.py new file mode 100644 index 00000000000..6aa8ad95e4b --- /dev/null +++ b/lib/galaxy/tools/toolbox/integrated_panel.py @@ -0,0 +1,87 @@ +import os +import shutil +import tempfile + +from .panel import ToolPanelElements +from .panel import panel_item_types + + +INTEGRATED_TOOL_PANEL_DESCRIPTION = """ +This is Galaxy's integrated tool panel and should be modified directly only for +reordering tools inside a section. Each time Galaxy starts up, this file is +synchronized with the various tool config files: tools, sections and labels +added to one of these files, will be added also here in the appropriate place, +while elements removed from the tool config files will be correspondingly +deleted from this file. +To modify locally managed tools (e.g. from tool_conf.xml) modify that file +directly and restart Galaxy. Whenever possible Tool Shed managed tools (e.g. +from shed_tool_conf.xml) should be managed from within the Galaxy interface or +via its API - but if changes are necessary (such as to hide a tool or re-assign +its section) modify that file and restart Galaxy. +""" + + +class ManagesIntegratedToolPanelMixin: + + def _init_integrated_tool_panel(self, config): + self.update_integrated_tool_panel = config.update_integrated_tool_panel + self._integrated_tool_panel_config = config.integrated_tool_panel_config + # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. + self._integrated_tool_panel = ToolPanelElements() + self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 + if self._integrated_tool_panel_config_has_contents: + self._load_integrated_tool_panel_keys() + + def _save_integrated_tool_panel(self): + if self.update_integrated_tool_panel: + # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. + # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel + # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel + # will be False when things like functional tests are the caller. + self._write_integrated_tool_panel_config_file() + + def _write_integrated_tool_panel_config_file( self ): + """ + Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators + use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. + """ + fd, filename = tempfile.mkstemp() + os.write( fd, '\n' ) + os.write( fd, '\n' ) + os.write( fd, ' \n') + for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): + if item: + if item_type == panel_item_types.TOOL: + os.write( fd, ' \n' % item.id ) + elif item_type == panel_item_types.WORKFLOW: + os.write( fd, ' \n' % item.id ) + elif item_type == panel_item_types.LABEL: + label_id = item.id or '' + label_text = item.text or '' + label_version = item.version or '' + os.write( fd, ' \n' ) + os.close( fd ) + shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) + os.chmod( self._integrated_tool_panel_config, 0644 ) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 03d12c9deee..6bdac195694 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -637,32 +637,6 @@ class AdminToolshed( AdminGalaxy ): message=message, status=status ) - @web.expose - @web.require_admin - def install_tool_shed_repositories( self, trans, tool_shed_repositories, reinstalling=False, **kwd ): - """Install specified tool shed repositories.""" - shed_tool_conf = kwd.get( 'shed_tool_conf', '' ) - tool_path = kwd[ 'tool_path' ] - install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) - # There must be a one-to-one mapping between items in the 3 lists: tool_shed_repositories, tool_panel_section_keys, repo_info_dicts. - tool_panel_section_keys = util.listify( kwd[ 'tool_panel_section_keys' ] ) - repo_info_dicts = util.listify( kwd[ 'repo_info_dicts' ] ) - irm = install_manager.InstallRepositoryManager( trans.app ) - for index, tool_shed_repository in enumerate( tool_shed_repositories ): - repo_info_dict = repo_info_dicts[ index ] - tool_panel_section_key = tool_panel_section_keys[ index ] - irm.install_tool_shed_repository( tool_shed_repository, - repo_info_dict, - tool_panel_section_key, - shed_tool_conf, - tool_path, - install_tool_dependencies, - reinstalling=reinstalling ) - tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='monitor_repository_installation', - tool_shed_repository_ids=tsr_ids_for_monitoring ) ) - @web.expose @web.require_admin def manage_repositories( self, trans, **kwd ): @@ -704,41 +678,21 @@ class AdminToolshed( AdminGalaxy ): reinstalling = util.string_as_bool( kwd.get( 'reinstalling', False ) ) encoded_kwd = kwd[ 'encoded_kwd' ] decoded_kwd = encoding_util.tool_shed_decode( encoded_kwd ) + install_tool_dependencies = CheckboxField.is_checked( decoded_kwd.get( 'install_tool_dependencies', '' ) ) tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ] - tool_panel_section_keys = decoded_kwd[ 'tool_panel_section_keys' ] - repo_info_dicts = decoded_kwd[ 'repo_info_dicts' ] - filtered_repo_info_dicts = [] - filtered_tool_panel_section_keys = [] - repositories_for_installation = [] - # Some repositories may have repository dependencies that are required to be installed before the - # dependent repository, so we'll order the list of tsr_ids to ensure all repositories install in the - # required order. - ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ - irm.order_components_for_installation( tsr_ids, - repo_info_dicts, - tool_panel_section_keys=tool_panel_section_keys ) - for tsr_id in ordered_tsr_ids: - repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .get( trans.security.decode_id( tsr_id ) ) - if repository.status in [ trans.install_model.ToolShedRepository.installation_status.NEW, - trans.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: - repositories_for_installation.append( repository ) - repo_info_dict, tool_panel_section_key = \ - irm.get_repository_components_for_installation( tsr_id, - ordered_tsr_ids, - ordered_repo_info_dicts, - ordered_tool_panel_section_keys ) - filtered_repo_info_dicts.append( repo_info_dict ) - filtered_tool_panel_section_keys.append( tool_panel_section_key ) - if repositories_for_installation: - decoded_kwd[ 'repo_info_dicts' ] = filtered_repo_info_dicts - decoded_kwd[ 'tool_panel_section_keys' ] = filtered_tool_panel_section_keys - self.install_tool_shed_repositories( trans, - repositories_for_installation, - reinstalling=reinstalling, - **decoded_kwd ) - else: - kwd[ 'message' ] = 'All selected tool shed repositories are already installed.' + decoded_kwd['install_tool_dependencies'] = install_tool_dependencies + try: + tool_shed_repositories = irm.install_repositories( + tsr_ids=tsr_ids, + decoded_kwd=decoded_kwd, + reinstalling=reinstalling, + ) + tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] + trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='monitor_repository_installation', + tool_shed_repository_ids=tsr_ids_for_monitoring ) ) + except install_manager.RepositoriesInstalledException as e: + kwd[ 'message' ] = e.message kwd[ 'status' ] = 'error' return self.repository_installation_grid( trans, **kwd ) @@ -1092,7 +1046,7 @@ class AdminToolshed( AdminGalaxy ): tool_path=tool_path, tool_shed_url=tool_shed_url ) created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - install_repository_manager.handle_tool_shed_repositories( installation_dict, using_api=False ) + install_repository_manager.handle_tool_shed_repositories( installation_dict ) if created_or_updated_tool_shed_repositories: installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories, filtered_repo_info_dicts=filtered_repo_info_dicts, diff --git a/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py b/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py index 8d56d04c540..5d829967d53 100644 --- a/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py +++ b/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py @@ -214,5 +214,5 @@ class CustomDatatypeLoader( object ): def load_installed_display_applications( self, installed_repository_dict, deactivate=False ): """Load or deactivate custom datatype display applications.""" - self.app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, + self.app.datatypes_registry.load_display_applications( self.app, installed_repository_dict=installed_repository_dict, deactivate=deactivate ) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index d955ed355e7..d0bb3fe16d0 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -7,6 +7,7 @@ import traceback from galaxy import exceptions from galaxy import eggs +from galaxy import util eggs.require( 'paramiko' ) eggs.require( 'ssh' ) @@ -454,10 +455,12 @@ class InstallRepositoryManager( object ): def get_repository_components_for_installation( self, encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys ): """ - The received encoded_tsr_ids, repo_info_dicts, and tool_panel_section_keys are 3 lists that - contain associated elements at each location in the list. This method will return the elements - from repo_info_dicts and tool_panel_section_keys associated with the received encoded_tsr_id - by determining its location in the received encoded_tsr_ids list. + The received encoded_tsr_ids, repo_info_dicts, and + tool_panel_section_keys are 3 lists that contain associated elements + at each location in the list. This method will return the elements + from repo_info_dicts and tool_panel_section_keys associated with the + received encoded_tsr_id by determining its location in the received + encoded_tsr_ids list. """ for index, tsr_id in enumerate( encoded_tsr_ids ): if tsr_id == encoded_tsr_id: @@ -621,9 +624,9 @@ class InstallRepositoryManager( object ): self.app.datatypes_registry.load_datatype_converters( self.app.toolbox, installed_repository_dict=repository_dict ) if display_path: # Load proprietary datatype display applications - self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + self.app.datatypes_registry.load_display_applications( self.app, installed_repository_dict=repository_dict ) - def handle_tool_shed_repositories( self, installation_dict, using_api=False ): + def handle_tool_shed_repositories( self, installation_dict ): # The following installation_dict entries are all required. install_repository_dependencies = installation_dict[ 'install_repository_dependencies' ] new_tool_panel_section_label = installation_dict[ 'new_tool_panel_section_label' ] @@ -699,40 +702,36 @@ class InstallRepositoryManager( object ): name, owner, changeset_revision ) - installed_tool_shed_repositories = self.__install_repositories( tool_shed_url, - repository_revision_dict, - repo_info_dicts, - install_options ) + installed_tool_shed_repositories = self.__initiate_and_install_repositories( + tool_shed_url, + repository_revision_dict, + repo_info_dicts, + install_options + ) return installed_tool_shed_repositories - def __install_repositories( self, tool_shed_url, repository_revision_dict, repo_info_dicts, install_options ): - # Keep track of all repositories that are installed - there may be more than one if repository dependencies are installed. - installed_tool_shed_repositories = [] + def __initiate_and_install_repositories( self, tool_shed_url, repository_revision_dict, repo_info_dicts, install_options ): try: has_repository_dependencies = repository_revision_dict[ 'has_repository_dependencies' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'has_repository_dependencies'." ) try: includes_tools = repository_revision_dict[ 'includes_tools' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'includes_tools'." ) try: includes_tool_dependencies = repository_revision_dict[ 'includes_tool_dependencies' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'includes_tool_dependencies'." ) try: includes_tools_for_display_in_tool_panel = repository_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'includes_tools_for_display_in_tool_panel'." ) # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain the repository information. install_repository_dependencies = install_options.get( 'install_repository_dependencies', False ) install_tool_dependencies = install_options.get( 'install_tool_dependencies', False ) if install_tool_dependencies: - if self.app.config.tool_dependency_dir is None: - no_tool_dependency_dir_message = "Tool dependencies can be automatically installed only if you set " - no_tool_dependency_dir_message += "the value of your 'tool_dependency_dir' setting in your Galaxy " - no_tool_dependency_dir_message += "configuration file (galaxy.ini) and restart your Galaxy server. " - raise exceptions.ConfigDoesNotAllowException( no_tool_dependency_dir_message ) + self.__assert_can_install_dependencies() new_tool_panel_section_label = install_options.get( 'new_tool_panel_section_label', '' ) shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: @@ -759,7 +758,7 @@ class InstallRepositoryManager( object ): tool_shed_url=tool_shed_url ) # Create the tool_shed_repository database records and gather additional information for repository installation. created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - self.handle_tool_shed_repositories( installation_dict, using_api=True ) + self.handle_tool_shed_repositories( installation_dict ) if created_or_updated_tool_shed_repositories: # Build the dictionary of information necessary for installing the repositories. installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories, @@ -778,37 +777,74 @@ class InstallRepositoryManager( object ): tool_panel_section_keys=tool_panel_section_keys, tool_path=tool_path, tool_shed_url=tool_shed_url ) - # Prepare the repositories for installation. Even though this method receives a single combination - # of tool_shed_url, name, owner and changeset_revision, there may be multiple repositories for installation - # at this point because repository dependencies may have added additional repositories for installation - # along with the single specified repository. + # Prepare the repositories for installation. Even though this + # method receives a single combination of tool_shed_url, name, + # owner and changeset_revision, there may be multiple repositories + # for installation at this point because repository dependencies + # may have added additional repositories for installation along + # with the single specified repository. encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = \ self.initiate_repository_installation( installation_dict ) - # Some repositories may have repository dependencies that are required to be installed before the - # dependent repository, so we'll order the list of tsr_ids to ensure all repositories install in - # the required order. + # Some repositories may have repository dependencies that are + # required to be installed before the dependent repository, so + # we'll order the list of tsr_ids to ensure all repositories + # install in the required order. tsr_ids = [ self.app.security.encode_id( tool_shed_repository.id ) for tool_shed_repository in tool_shed_repositories ] - ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ - self.order_components_for_installation( tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) - # Install the repositories, keeping track of each one for later display. - for index, tsr_id in enumerate( ordered_tsr_ids ): - tool_shed_repository = self.install_model.context.query( self.install_model.ToolShedRepository ) \ - .get( self.app.security.decode_id( tsr_id ) ) - if tool_shed_repository.status in [ self.install_model.ToolShedRepository.installation_status.NEW, - self.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: - repo_info_dict = ordered_repo_info_dicts[ index ] - tool_panel_section_key = ordered_tool_panel_section_keys[ index ] - self.install_tool_shed_repository( tool_shed_repository, - repo_info_dict, - tool_panel_section_key, - shed_tool_conf, - tool_path, - install_tool_dependencies, - reinstalling=False ) - installed_tool_shed_repositories.append( tool_shed_repository ) + + decoded_kwd = dict( + shed_tool_conf=shed_tool_conf, + tool_path=tool_path, + tool_panel_section_keys=tool_panel_section_keys, + repo_info_dicts=repo_info_dicts, + install_tool_dependencies=install_tool_dependencies, + ) + return self.install_repositories(tsr_ids, decoded_kwd, reinstalling=False) + + def install_repositories( self, tsr_ids, decoded_kwd, reinstalling ): + shed_tool_conf = decoded_kwd.get( 'shed_tool_conf', '' ) + tool_path = decoded_kwd[ 'tool_path' ] + tool_panel_section_keys = util.listify( decoded_kwd[ 'tool_panel_section_keys' ] ) + repo_info_dicts = util.listify( decoded_kwd[ 'repo_info_dicts' ] ) + install_tool_dependencies = decoded_kwd['install_tool_dependencies'] + filtered_repo_info_dicts = [] + filtered_tool_panel_section_keys = [] + repositories_for_installation = [] + # Some repositories may have repository dependencies that are required to be installed before the + # dependent repository, so we'll order the list of tsr_ids to ensure all repositories install in the + # required order. + ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ + self.order_components_for_installation( tsr_ids, + repo_info_dicts, + tool_panel_section_keys=tool_panel_section_keys ) + for tsr_id in ordered_tsr_ids: + repository = self.install_model.context.query( self.install_model.ToolShedRepository ) \ + .get( self.app.security.decode_id( tsr_id ) ) + if repository.status in [ self.install_model.ToolShedRepository.installation_status.NEW, + self.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: + repositories_for_installation.append( repository ) + repo_info_dict, tool_panel_section_key = \ + self.get_repository_components_for_installation( tsr_id, + ordered_tsr_ids, + ordered_repo_info_dicts, + ordered_tool_panel_section_keys ) + filtered_repo_info_dicts.append( repo_info_dict ) + filtered_tool_panel_section_keys.append( tool_panel_section_key ) + + installed_tool_shed_repositories = [] + if repositories_for_installation: + for index, tool_shed_repository in enumerate( repositories_for_installation ): + repo_info_dict = filtered_repo_info_dicts[ index ] + tool_panel_section_key = filtered_tool_panel_section_keys[ index ] + self.install_tool_shed_repository( tool_shed_repository, + repo_info_dict=repo_info_dict, + tool_panel_section_key=tool_panel_section_key, + shed_tool_conf=shed_tool_conf, + tool_path=tool_path, + install_tool_dependencies=install_tool_dependencies, + reinstalling=reinstalling ) + installed_tool_shed_repositories.append( tool_shed_repository ) else: - # We're attempting to install more than 1 repository, and all of them have already been installed. - raise exceptions.RequestParameterInvalidException( 'All repositories that you are attempting to install have been previously installed.' ) + raise RepositoriesInstalledException() return installed_tool_shed_repositories def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, @@ -957,6 +993,19 @@ class InstallRepositoryManager( object ): self.install_model.context.add( tool_shed_repository ) self.install_model.context.flush() + def __assert_can_install_dependencies(self): + if self.app.config.tool_dependency_dir is None: + no_tool_dependency_dir_message = "Tool dependencies can be automatically installed only if you set " + no_tool_dependency_dir_message += "the value of your 'tool_dependency_dir' setting in your Galaxy " + no_tool_dependency_dir_message += "configuration file (galaxy.ini) and restart your Galaxy server. " + raise exceptions.ConfigDoesNotAllowException( no_tool_dependency_dir_message ) + + +class RepositoriesInstalledException(exceptions.RequestParameterInvalidException): + + def __init__(self): + super(RepositoriesInstalledException, self).__init__('All repositories that you are attempting to install have been previously installed.') + def fetch_tool_versions( app, tool_shed_repository ): """ Fetch a data structure describing tool shed versions from the tool shed diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index f8469fe96f8..e8cbcc1db54 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -523,7 +523,7 @@ class ToolMigrationManager( object ): installed_repository_dict=repository_dict ) if display_path: # Load proprietary datatype display applications - self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + self.app.datatypes_registry.load_display_applications( self.app, installed_repository_dict=repository_dict ) basic_util.remove_dir( work_dir ) def install_repository( self, repository_elem, tool_shed_repository, install_dependencies, is_repository_dependency=False ): diff --git a/static/scripts/mvc/tools/tools-form-base.js b/static/scripts/mvc/tools/tools-form-base.js index 94a0bfece5f..0da0f7a1a32 100644 --- a/static/scripts/mvc/tools/tools-form-base.js +++ b/static/scripts/mvc/tools/tools-form-base.js @@ -108,7 +108,7 @@ define(['utils/utils', 'utils/deferred', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc', var new_state = self.tree.finalize(); if (!_.isEqual(new_state, current_state)) { current_state = new_state; - self._updateModel($.extend({}, current_state)); + self._updateModel($.extend(true, {}, current_state)); } }); }); diff --git a/static/scripts/packed/mvc/tools/tools-form-base.js b/static/scripts/packed/mvc/tools/tools-form-base.js index 3257db4e800..b56901ab363 100644 --- a/static/scripts/packed/mvc/tools/tools-form-base.js +++ b/static/scripts/packed/mvc/tools/tools-form-base.js @@ -1 +1 @@ -define(["utils/utils","utils/deferred","mvc/ui/ui-portlet","mvc/ui/ui-misc","mvc/citation/citation-model","mvc/citation/citation-view","mvc/tools","mvc/tools/tools-template","mvc/tools/tools-section","mvc/tools/tools-tree"],function(f,g,e,j,h,a,d,c,i,b){return Backbone.View.extend({initialize:function(k){this.optionsDefault={is_dynamic:true,narrow:false,initial_errors:false,cls_portlet:"ui-portlet-limited"};this.options=f.merge(k,this.optionsDefault);console.debug(this.options);var l=parent.Galaxy;if(l&&l.modal){this.modal=l.modal}else{this.modal=new j.Modal.View()}if(l&&l.currUser){this.is_admin=l.currUser.get("is_admin")}else{this.is_admin=false}this.container=this.options.container||"body";this.deferred=new g();this.setElement("
");$(this.container).append(this.$el);this.build(this.options)},build:function(m){var k=this;this.off("refresh");this.off("reset");this.field_list={};this.input_list={};this.element_list={};this.tree=new b(this);this.history={};k.options.inputs=m&&m.inputs;this._renderForm(m);var l=this.tree.finalize();if(m.initial_errors){this._errors(m)}this.on("refresh",function(){k.deferred.reset();k.deferred.execute(function(){var n=k.tree.finalize();if(!_.isEqual(n,l)){l=n;k._updateModel($.extend({},l))}})});this.on("reset",function(){for(var n in this.element_list){this.element_list[n].reset()}})},reciept:function(k){$(this.container).empty();$(this.container).append(k)},highlight:function(l,m,k){var n=this.element_list[l];if(n){n.error(m||"Please verify this parameter.");if(!k){$("html, body").animate({scrollTop:n.$el.offset().top-20},500)}}},_errors:function(m){this.trigger("reset");if(m&&m.errors){var n=this.tree.matchResponse(m.errors);for(var l in this.element_list){var k=this.element_list[l];if(n[l]){this.highlight(l,n[l],true)}}}},_renderForm:function(s){var r=this;this.message=new j.Message();var l=new j.ButtonMenu({icon:"fa-cubes",title:(!s.narrow&&"Versions")||null,tooltip:"Select another tool version"});if(s.versions&&s.versions.length>1){for(var n in s.versions){var p=s.versions[n];if(p!=s.version){l.addMenu({title:"Switch to "+p,version:p,icon:"fa-cube",onclick:function(){r.options.id=r.options.id.replace(r.options.version,this.version);r.options.version=this.version;r.deferred.reset();r.deferred.execute(function(){r._buildModel()})}})}}}else{l.$el.hide()}var o=new j.ButtonMenu({icon:"fa-caret-down",title:(!s.narrow&&"Options")||null,tooltip:"View available options"});if(s.biostar_url){o.addMenu({icon:"fa-question-circle",title:"Question?",tooltip:"Ask a question about this tool (Biostar)",onclick:function(){window.open(s.biostar_url+"/p/new/post/")}});o.addMenu({icon:"fa-search",title:"Search",tooltip:"Search help for this tool (Biostar)",onclick:function(){window.open(s.biostar_url+"/t/"+s.id+"/")}})}o.addMenu({icon:"fa-share",title:"Share",tooltip:"Share this tool",onclick:function(){prompt("Copy to clipboard: Ctrl+C, Enter",window.location.origin+galaxy_config.root+"root?tool_id="+s.id)}});if(this.is_admin){o.addMenu({icon:"fa-download",title:"Download",tooltip:"Download this tool",onclick:function(){window.location.href=galaxy_config.root+"api/tools/"+s.id+"/download"}})}if(s.requirements&&s.requirements.length>0){o.addMenu({icon:"fa-info-circle",title:"Requirements",tooltip:"Display tool requirements",onclick:function(){if(!this.visible){this.visible=true;r.message.update({persistent:true,message:c.requirements(s),status:"info"})}else{this.visible=false;r.message.update({message:""})}}})}if(this.options.sharable_url){o.addMenu({icon:"fa-external-link",title:"See in Tool Shed",tooltip:"Access the repository",onclick:function(){window.open(r.options.sharable_url)}})}this.section=new i.View(r,{inputs:s.inputs});if(this.incompatible){this.$el.hide();$("#tool-form-classic").show();return}this.portlet=new e.View({icon:"fa-wrench",title:""+s.name+" "+s.description+" (Galaxy Tool Version "+s.version+")",cls:this.options.cls_portlet,operations:{menu:o,versions:l},buttons:this.buttons});this.portlet.append(this.message.$el.addClass("ui-margin-top"));this.portlet.append(this.section.$el);this.$el.empty();this.$el.append(this.portlet.$el);if(s.help!=""){this.$el.append(c.help(s.help))}if(s.citations){var q=$("
");var k=new h.ToolCitationCollection();k.tool_id=s.id;var m=new a.CitationListView({el:q,collection:k});m.render();k.fetch();this.$el.append(q)}if(s.message){this.message.update({persistent:true,status:"warning",message:s.message})}console.debug("tools-form-base::initialize() - Completed.")}})}); \ No newline at end of file +define(["utils/utils","utils/deferred","mvc/ui/ui-portlet","mvc/ui/ui-misc","mvc/citation/citation-model","mvc/citation/citation-view","mvc/tools","mvc/tools/tools-template","mvc/tools/tools-section","mvc/tools/tools-tree"],function(f,g,e,j,h,a,d,c,i,b){return Backbone.View.extend({initialize:function(k){this.optionsDefault={is_dynamic:true,narrow:false,initial_errors:false,cls_portlet:"ui-portlet-limited"};this.options=f.merge(k,this.optionsDefault);console.debug(this.options);var l=parent.Galaxy;if(l&&l.modal){this.modal=l.modal}else{this.modal=new j.Modal.View()}if(l&&l.currUser){this.is_admin=l.currUser.get("is_admin")}else{this.is_admin=false}this.container=this.options.container||"body";this.deferred=new g();this.setElement("
");$(this.container).append(this.$el);this.build(this.options)},build:function(m){var k=this;this.off("refresh");this.off("reset");this.field_list={};this.input_list={};this.element_list={};this.tree=new b(this);this.history={};k.options.inputs=m&&m.inputs;this._renderForm(m);var l=this.tree.finalize();if(m.initial_errors){this._errors(m)}this.on("refresh",function(){k.deferred.reset();k.deferred.execute(function(){var n=k.tree.finalize();if(!_.isEqual(n,l)){l=n;k._updateModel($.extend(true,{},l))}})});this.on("reset",function(){for(var n in this.element_list){this.element_list[n].reset()}})},reciept:function(k){$(this.container).empty();$(this.container).append(k)},highlight:function(l,m,k){var n=this.element_list[l];if(n){n.error(m||"Please verify this parameter.");if(!k){$("html, body").animate({scrollTop:n.$el.offset().top-20},500)}}},_errors:function(m){this.trigger("reset");if(m&&m.errors){var n=this.tree.matchResponse(m.errors);for(var l in this.element_list){var k=this.element_list[l];if(n[l]){this.highlight(l,n[l],true)}}}},_renderForm:function(s){var r=this;this.message=new j.Message();var l=new j.ButtonMenu({icon:"fa-cubes",title:(!s.narrow&&"Versions")||null,tooltip:"Select another tool version"});if(s.versions&&s.versions.length>1){for(var n in s.versions){var p=s.versions[n];if(p!=s.version){l.addMenu({title:"Switch to "+p,version:p,icon:"fa-cube",onclick:function(){r.options.id=r.options.id.replace(r.options.version,this.version);r.options.version=this.version;r.deferred.reset();r.deferred.execute(function(){r._buildModel()})}})}}}else{l.$el.hide()}var o=new j.ButtonMenu({icon:"fa-caret-down",title:(!s.narrow&&"Options")||null,tooltip:"View available options"});if(s.biostar_url){o.addMenu({icon:"fa-question-circle",title:"Question?",tooltip:"Ask a question about this tool (Biostar)",onclick:function(){window.open(s.biostar_url+"/p/new/post/")}});o.addMenu({icon:"fa-search",title:"Search",tooltip:"Search help for this tool (Biostar)",onclick:function(){window.open(s.biostar_url+"/t/"+s.id+"/")}})}o.addMenu({icon:"fa-share",title:"Share",tooltip:"Share this tool",onclick:function(){prompt("Copy to clipboard: Ctrl+C, Enter",window.location.origin+galaxy_config.root+"root?tool_id="+s.id)}});if(this.is_admin){o.addMenu({icon:"fa-download",title:"Download",tooltip:"Download this tool",onclick:function(){window.location.href=galaxy_config.root+"api/tools/"+s.id+"/download"}})}if(s.requirements&&s.requirements.length>0){o.addMenu({icon:"fa-info-circle",title:"Requirements",tooltip:"Display tool requirements",onclick:function(){if(!this.visible){this.visible=true;r.message.update({persistent:true,message:c.requirements(s),status:"info"})}else{this.visible=false;r.message.update({message:""})}}})}if(this.options.sharable_url){o.addMenu({icon:"fa-external-link",title:"See in Tool Shed",tooltip:"Access the repository",onclick:function(){window.open(r.options.sharable_url)}})}this.section=new i.View(r,{inputs:s.inputs});if(this.incompatible){this.$el.hide();$("#tool-form-classic").show();return}this.portlet=new e.View({icon:"fa-wrench",title:""+s.name+" "+s.description+" (Galaxy Tool Version "+s.version+")",cls:this.options.cls_portlet,operations:{menu:o,versions:l},buttons:this.buttons});this.portlet.append(this.message.$el.addClass("ui-margin-top"));this.portlet.append(this.section.$el);this.$el.empty();this.$el.append(this.portlet.$el);if(s.help!=""){this.$el.append(c.help(s.help))}if(s.citations){var q=$("
");var k=new h.ToolCitationCollection();k.tool_id=s.id;var m=new a.CitationListView({el:q,collection:k});m.render();k.fetch();this.$el.append(q)}if(s.message){this.message.update({persistent:true,status:"warning",message:s.message})}console.debug("tools-form-base::initialize() - Completed.")}})}); \ No newline at end of file diff --git a/static/scripts/templates/tool_search.handlebars b/static/scripts/templates/tool_search.handlebars index bd035f2e92b..995b9ce4af4 100644 --- a/static/scripts/templates/tool_search.handlebars +++ b/static/scripts/templates/tool_search.handlebars @@ -1,3 +1,3 @@ - + - \ No newline at end of file + diff --git a/tool-data/shared/igv/igv_build_sites.txt.sample b/tool-data/shared/igv/igv_build_sites.txt.sample index 77d262da4f6..ce3f18f16b6 100644 --- a/tool-data/shared/igv/igv_build_sites.txt.sample +++ b/tool-data/shared/igv/igv_build_sites.txt.sample @@ -1,4 +1,4 @@ #site_id site_name site_url dbkey ivg_build_name -web_link_main web current http://www.broadinstitute.org/igv/projects/current/igv.php hg19,hg_g1k_v37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum hg19,b37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum +web_link_main web current http://www.broadinstitute.org/igv/projects/current/igv.php hg_g1k_v37 b37 #web_jnlp_1.5 web 1.5 http://www.broadinstitute.org/igvdata/jws/prod hg19,hg_g1k_v37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum hg19,b37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum -#local_default local http://localhost:60151/load hg19,hg_g1k_v37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum hg19,b37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum \ No newline at end of file +#local_default local http://localhost:60151/load hg19,hg_g1k_v37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum hg19,b37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum