opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixes for installing tool shed repositories that include proprietary datatypes.

Browse Source
This commit is contained in:
Greg Von Kuster
2012-07-10 11:38:32 -04:00
parent 9894424833
commit 9949542c34
6 changed files with 90 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tool_shed/install_manager.py
+5 -3
View File
@@ -190,9 +190,11 @@ class InstallManager( object ):
except:
pass
if 'datatypes' in metadata_dict:
update_tool_shed_repository_status( self.app,
tool_shed_repository,
self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES )
tool_shed_repository.status = self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
if not tool_shed_repository.includes_datatypes:
tool_shed_repository.includes_datatypes = True
self.app.sa_session.add( tool_shed_repository )
self.app.sa_session.flush()
work_dir = make_tmp_directory()
datatypes_config = get_config_from_repository( self.app,
'datatypes_conf.xml',
lib/galaxy/util/shed_util.py
+9
View File
@@ -309,6 +309,15 @@ def copy_sample_files( app, sample_files, sample_files_copied=None, dest_path=No
for filename in sample_files:
if filename not in sample_files_copied:
copy_sample_file( app, filename, dest_path=dest_path )
def create_repo_info_dict( repository, owner, repository_clone_url, changeset_revision, ctx_rev, metadata ):
repo_info_dict = {}
repo_info_dict[ repository.name ] = ( repository.description,
repository_clone_url,
changeset_revision,
ctx_rev,
owner,
metadata.get( 'tool_dependencies', None ) )
return repo_info_dict
def create_repository_dict_for_proprietary_datatypes( tool_shed, name, owner, installed_changeset_revision, tool_dicts, converter_path=None, display_path=None ):
return dict( tool_shed=tool_shed,
repository_name=name,
lib/galaxy/web/base/controller.py
+24 -1
View File
@@ -1,7 +1,7 @@
"""
Contains functionality needed in every web interface
"""
import os, time, logging, re, string, sys, glob, shutil, tempfile, subprocess
import os, time, logging, re, string, sys, glob, shutil, tempfile, subprocess, operator
from datetime import date, datetime, timedelta
from time import strftime
from galaxy import config, tools, web, util
@@ -177,6 +177,14 @@ class BaseAPIController( BaseController ):
def not_implemented( self, trans, **kwd ):
raise HTTPNotImplemented()
class Datatype( object ):
"""Used for storing in-memory list of datatypes currently in the datatypes registry."""
def __init__( self, extension, dtype, type_extension, mimetype, display_in_upload ):
self.extension = extension
self.dtype = dtype
self.type_extension = type_extension
self.mimetype = mimetype
self.display_in_upload = display_in_upload
#
# -- Mixins for working with Galaxy objects. --
#
@@ -2722,3 +2730,18 @@ def get_webapp( trans, **kwd ):
return trans.environ[ 'webapp' ]
# The default is galaxy.
return 'galaxy'
def sort_by_attr( seq, attr ):
"""
Sort the sequence of objects by object's attribute
Arguments:
seq - the list or any sequence (including immutable one) of objects to sort.
attr - the name of attribute to sort by
"""
# Use the "Schwartzian transform"
# Create the auxiliary list of tuples where every i-th tuple has form
# (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not
# only to provide stable sorting, but mainly to eliminate comparison of objects
# (which can be expensive or prohibited) in case of equal attribute values.
intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq )
intermed.sort()
return map( operator.getitem, intermed, ( -1, ) * len( intermed ) )
lib/galaxy/web/controllers/admin_toolshed.py
+23 -51
View File
@@ -605,7 +605,8 @@ class AdminToolshed( AdminGalaxy ):
tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ]
else:
tool_section = None
for tool_shed_repository, repo_info_dict in zip( tool_shed_repositories, repo_info_dicts ):
for tup in zip( tool_shed_repositories, repo_info_dicts ):
tool_shed_repository, repo_info_dict = tup
# Clone each repository to the configured location.
update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING )
repo_info_tuple = repo_info_dict[ tool_shed_repository.name ]
@@ -670,37 +671,6 @@ class AdminToolshed( AdminGalaxy ):
shutil.rmtree( work_dir )
except:
pass
if 'datatypes' in metadata:
update_tool_shed_repository_status( trans.app,
tool_shed_repository,
trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES )
work_dir = make_tmp_directory()
datatypes_config = get_config_from_repository( trans.app,
'datatypes_conf.xml',
tool_shed_repository,
tool_shed_repository.installed_changeset_revision,
work_dir )
# Load proprietary data types required by tools.
converter_path, display_path = alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, relative_install_dir, override=False )
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed,
name=tool_shed_repository.name,
owner=tool_shed_repository.owner,
installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dicts=metadata.get( 'tools', [] ),
converter_path=converter_path,
display_path=display_path )
if converter_path:
# Load proprietary datatype converters
trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict )
if display_path:
# Load proprietary datatype display applications
trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict )
try:
shutil.rmtree( work_dir )
except:
pass
update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED )
tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ]
return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
@@ -749,9 +719,11 @@ class AdminToolshed( AdminGalaxy ):
except:
pass
if 'datatypes' in metadata_dict:
update_tool_shed_repository_status( trans.app,
tool_shed_repository,
trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES )
tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
if not tool_shed_repository.includes_datatypes:
tool_shed_repository.includes_datatypes = True
trans.sa_session.add( tool_shed_repository )
trans.sa_session.flush()
work_dir = make_tmp_directory()
datatypes_config = get_config_from_repository( trans.app,
'datatypes_conf.xml',
@@ -1255,13 +1227,12 @@ class AdminToolshed( AdminGalaxy ):
repo_info_dict = kwd.get( 'repo_info_dict', None )
if not repo_info_dict:
# This should only happen if the tool_shed_repository does not include any valid tools.
repo_info_dict = {}
repo_info_dict[ tool_shed_repository.name ] = ( tool_shed_repository.description,
repository_clone_url,
tool_shed_repository.installed_changeset_revision,
ctx_rev,
tool_shed_repository.owner,
metadata.get( 'tool_dependencies', None ) )
repo_info_dict = create_repo_info_dict( tool_shed_repository,
tool_shed_repository.owner,
repository_clone_url,
tool_shed_repository.installed_changeset_revision,
ctx_rev,
metadata )
new_kwd = dict( includes_tool_dependencies=tool_shed_repository.includes_tool_dependencies,
includes_tools=tool_shed_repository.includes_tools,
install_tool_dependencies=install_tool_dependencies,
@@ -1291,7 +1262,8 @@ class AdminToolshed( AdminGalaxy ):
if ids is not None and status_list is not None:
ids = util.listify( ids )
status_list = util.listify( status_list )
for id, status in zip( ids, status_list ):
for tup in zip( ids, status_list ):
id, status = tup
repository = trans.sa_session.query( trans.model.ToolShedRepository ).get( trans.security.decode_id( id ) )
if repository.status != status:
rval.append( dict( id=id,
@@ -1335,13 +1307,12 @@ class AdminToolshed( AdminGalaxy ):
tool_shed_url = get_url_from_repository_tool_shed( trans.app, repository )
ctx_rev = get_ctx_rev( tool_shed_url, repository.name, repository.owner, repository.installed_changeset_revision )
repository_clone_url = generate_clone_url( trans, repository )
repo_info_dict = {}
repo_info_dict[ repository.name ] = ( repository.description,
repository_clone_url,
repository.installed_changeset_revision,
ctx_rev,
repository.owner,
metadata.get( 'tool_dependencies', None ) )
repo_info_dict = create_repo_info_dict( repository,
repository.owner,
repository_clone_url,
repository.installed_changeset_revision,
ctx_rev,
metadata )
# Get the location in the tool panel in which the tool was originally loaded.
if 'tool_panel_section' in metadata:
tool_panel_dict = metadata[ 'tool_panel_section' ]
@@ -1419,7 +1390,8 @@ class AdminToolshed( AdminGalaxy ):
if ids is not None and status_list is not None:
ids = util.listify( ids )
status_list = util.listify( status_list )
for id, status in zip( ids, status_list ):
for tup in zip( ids, status_list ):
id, status = tup
tool_dependency = trans.sa_session.query( trans.model.ToolDependency ).get( trans.security.decode_id( id ) )
if tool_dependency.status != status:
rval.append( dict( id=id,
lib/galaxy/webapps/community/controllers/repository.py
+5 -34
View File
@@ -9,7 +9,7 @@ from galaxy.webapps.community.model import directory_hash_id
from galaxy.web.framework.helpers import time_ago, iff, grids
from galaxy.util.json import from_json_string, to_json_string
from galaxy.model.orm import *
from galaxy.util.shed_util import get_changectx_for_changeset, get_configured_ui, get_repository_file_contents, make_tmp_directory, NOT_TOOL_CONFIGS
from galaxy.util.shed_util import create_repo_info_dict, get_changectx_for_changeset, get_configured_ui, get_repository_file_contents, make_tmp_directory, NOT_TOOL_CONFIGS
from galaxy.util.shed_util import open_repository_files_folder, reversed_lower_upper_bounded_changelog, strip_path
from galaxy.tool_shed.encoding_util import *
from common import *
@@ -931,29 +931,6 @@ class RepositoryController( BaseUIController, ItemRatings ):
trans.sa_session.flush()
download_url = '/repos/%s/%s/archive/%s' % ( repository.user.username, repository.name, file_type_str )
return trans.response.send_redirect( download_url )
def __encode_repo_info_dict( self, trans, repository_metadata_ids ):
repo_info_dict = {}
includes_tools = False
includes_tool_dependencies = False
for repository_metadata_id in util.listify( repository_metadata_ids ):
repository_metadata = get_repository_metadata_by_id( trans, repository_metadata_id )
if not includes_tools and 'tools' in repository_metadata.metadata:
includes_tools = True
if not includes_tool_dependencies and 'tool_dependencies' in repository_metadata.metadata:
includes_tool_dependencies = True
repository = get_repository( trans, trans.security.encode_id( repository_metadata.repository_id ) )
# Get the changelog rev for this changeset_revision.
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
changeset_revision = repository_metadata.changeset_revision
ctx = get_changectx_for_changeset( repo, changeset_revision )
repository_id = trans.security.encode_id( repository.id )
repository_clone_url = generate_clone_url( trans, repository_id )
if includes_tool_dependencies:
repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision, str( ctx.rev() ) )
else:
repo_info_dict[ repository.name ] = ( repository.description, repository_clone_url, changeset_revision, str( ctx.rev() ) )
return tool_shed_encode( repo_info_dict ), includes_tools, includes_tool_dependencies
@web.expose
def find_tools( self, trans, **kwd ):
params = util.Params( kwd )
@@ -1156,7 +1133,8 @@ class RepositoryController( BaseUIController, ItemRatings ):
includes_tools = False
includes_tool_dependencies = False
repo_info_dicts = []
for repository_id, changeset_revision in zip( util.listify( repository_ids ), util.listify( changeset_revisions ) ):
for tup in zip( util.listify( repository_ids ), util.listify( changeset_revisions ) ):
repository_id, changeset_revision = tup
repository_clone_url = generate_clone_url( trans, repository_id )
repository = get_repository( trans, repository_id )
repository_metadata = get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision )
@@ -1168,15 +1146,8 @@ class RepositoryController( BaseUIController, ItemRatings ):
repo_dir = repository.repo_path
repo = hg.repository( get_configured_ui(), repo_dir )
ctx = get_changectx_for_changeset( repo, changeset_revision )
repo_info_dict = {}
repo_info_dict[ repository.name ] = ( repository.description,
repository_clone_url,
changeset_revision,
str( ctx.rev() ),
repository.user.username,
metadata.get( 'tool_dependencies', None ) )
encoded_repo_info_dict = tool_shed_encode( repo_info_dict )
repo_info_dicts.append( encoded_repo_info_dict )
repo_info_dict = create_repo_info_dict( repository, repository.user.username, repository_clone_url, changeset_revision, str( ctx.rev() ), metadata )
repo_info_dicts.append( tool_shed_encode( repo_info_dict ) )
return dict( includes_tools=includes_tools, includes_tool_dependencies=includes_tool_dependencies, repo_info_dicts=repo_info_dicts )
@web.expose
def get_readme( self, trans, **kwd ):
templates/admin/view_datatypes_registry.mako
+24 -18
View File
@@ -5,8 +5,24 @@
${render_msg( message, status )}
%endif
<%
import galaxy.util
from galaxy.web.base.controller import sort_by_attr, Datatype
ctr = 0
datatypes = []
for elem in trans.app.datatypes_registry.datatype_elems:
# Build a list of objects that can be sorted.
extension = elem.get( 'extension', None )
dtype = elem.get( 'type', None )
type_extension = elem.get( 'type_extension', None )
mimetype = elem.get( 'mimetype', None )
display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) )
datatypes.append( Datatype( extension, dtype, type_extension, mimetype, display_in_upload ) )
sorted_datatypes = sort_by_attr( datatypes, 'extension' )
%>
<div class="toolForm">
<div class="toolFormTitle">Current datatypes registry contains ${len( trans.app.datatypes_registry.datatype_elems )} datatypes</div>
<div class="toolFormTitle">Current datatypes registry contains ${len( sorted_datatypes )} datatypes</div>
<div class="toolFormBody">
<table class="manage-table colored" border="0" cellspacing="0" cellpadding="0" width="100%">
<tr>
@@ -14,33 +30,23 @@
<th>Type</th>
<th>Mimetype</th>
<th>Display in upload</th>
<th>Datatype class</th>
</tr>
<% ctr = 0 %>
%for elem in trans.app.datatypes_registry.datatype_elems:
<%
import galaxy.util
extension = elem.get( 'extension', None )
dtype = elem.get( 'type', None )
type_extension = elem.get( 'type_extension', None )
mimetype = elem.get( 'mimetype', None )
display_in_upload = galaxy.util.string_as_bool( elem.get( 'display_in_upload', False ) )
%>
%for datatype in sorted_datatypes:
%if ctr % 2 == 1:
<tr class="odd_row">
%else:
<tr class="tr">
%endif
<td>${extension}</td>
<td>${dtype}</td>
<td>${datatype.extension}</td>
<td>${datatype.dtype}</td>
<td>
%if mimetype:
${mimetype}
%if datatype.mimetype:
${datatype.mimetype}
%endif
</td>
<td>
%if display_in_upload:
${display_in_upload}
%if datatype.display_in_upload:
${datatype.display_in_upload}
%endif
</td>
</tr>