opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Eliminate the use of Galaxy's elementree egg within the tool shed.

Browse Source
This commit is contained in:
Greg Von Kuster
2013-05-28 12:24:52 -04:00
parent 5e912d442a
commit 8eb3e7d4fa
18 changed files with 283 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/tool_shed/galaxy_install/__init__.py
+3 -11
View File
@@ -5,12 +5,7 @@ import os, logging
import tool_shed.util.shed_util_common
import tool_shed.util.datatype_util
from galaxy.model.orm import and_
from galaxy import eggs
import pkg_resources
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree, ElementInclude
from tool_shed.util import xml_util
log = logging.getLogger( __name__ )
@@ -25,13 +20,10 @@ class InstalledRepositoryManager( object ):
self.installed_repository_dicts = []
def get_repository_install_dir( self, tool_shed_repository ):
for tool_config in self.tool_configs:
try:
tree = ElementTree.parse( tool_config )
except Exception, e:
log.debug( "Exception attempting to parse %s: %s" % ( str( tool_config ), str( e ) ) )
tree, error_message = xml_util.parse_xml( tool_config )
if tree is None:
return None
root = tree.getroot()
ElementInclude.include( root )
tool_path = root.get( 'tool_path', None )
if tool_path:
ts = tool_shed.util.shed_util_common.clean_tool_shed_url( tool_shed_repository.tool_shed )
lib/tool_shed/galaxy_install/install_manager.py
+54 -44
View File
@@ -38,50 +38,56 @@ class InstallManager( object ):
self.proprietary_tool_confs = self.non_shed_tool_panel_configs
self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number )
# Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config.
tree = xml_util.parse_xml( migrated_tools_config )
root = tree.getroot()
self.tool_path = root.get( 'tool_path' )
print "Repositories will be installed into configured tool_path location ", str( self.tool_path )
# Parse tool_shed_install_config to check each of the tools.
self.tool_shed_install_config = tool_shed_install_config
tree = xml_util.parse_xml( tool_shed_install_config )
root = tree.getroot()
self.tool_shed = suc.clean_tool_shed_url( root.get( 'name' ) )
self.repository_owner = common_util.REPOSITORY_OWNER
index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config )
# Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in
# tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk.
# The default behavior is that the tool shed is down.
tool_shed_accessible = False
tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app )
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number )
tree, error_message = xml_util.parse_xml( migrated_tools_config )
if tree is None:
print error_message
else:
# It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but
# we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in
# the database.
tool_shed_accessible = True
missing_tool_configs_dict = odict()
if tool_shed_accessible:
if len( self.proprietary_tool_confs ) == 1:
plural = ''
file_names = self.proprietary_tool_confs[ 0 ]
root = tree.getroot()
self.tool_path = root.get( 'tool_path' )
print "Repositories will be installed into configured tool_path location ", str( self.tool_path )
# Parse tool_shed_install_config to check each of the tools.
self.tool_shed_install_config = tool_shed_install_config
tree, error_message = xml_util.parse_xml( tool_shed_install_config )
if tree is None:
print error_message
else:
plural = 's'
file_names = ', '.join( self.proprietary_tool_confs )
if missing_tool_configs_dict:
for repository_elem in root:
self.install_repository( repository_elem, install_dependencies )
else:
message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number )
message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names )
print message
else:
message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number )
message += "Try again later.\n"
print message
root = tree.getroot()
self.tool_shed = suc.clean_tool_shed_url( root.get( 'name' ) )
self.repository_owner = common_util.REPOSITORY_OWNER
index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config )
# Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in
# tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk.
# The default behavior is that the tool shed is down.
tool_shed_accessible = False
tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app )
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number )
else:
# It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but
# we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in
# the database.
tool_shed_accessible = True
missing_tool_configs_dict = odict()
if tool_shed_accessible:
if len( self.proprietary_tool_confs ) == 1:
plural = ''
file_names = self.proprietary_tool_confs[ 0 ]
else:
plural = 's'
file_names = ', '.join( self.proprietary_tool_confs )
if missing_tool_configs_dict:
for repository_elem in root:
self.install_repository( repository_elem, install_dependencies )
else:
message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number )
message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names )
print message
else:
message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number )
message += "Try again later.\n"
print message
def get_guid( self, repository_clone_url, relative_install_dir, tool_config ):
if self.shed_config_dict.get( 'tool_path' ):
@@ -108,7 +114,9 @@ class InstallManager( object ):
tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) )
# Parse the XML and load the file attributes for later checking against the integrated elements from self.proprietary_tool_confs.
migrated_tool_configs = []
tree = xml_util.parse_xml( tools_xml_file_path )
tree, error_message = xml_util.parse_xml( tools_xml_file_path )
if tree is None:
return []
root = tree.getroot()
for elem in root:
if elem.tag == 'repository':
@@ -117,7 +125,9 @@ class InstallManager( object ):
# Parse each file in self.proprietary_tool_confs and generate the integrated list of tool panel Elements that contain them.
tool_panel_elems = []
for proprietary_tool_conf in self.proprietary_tool_confs:
tree = xml_util.parse_xml( proprietary_tool_conf )
tree, error_message = xml_util.parse_xml( proprietary_tool_conf )
if tree is None:
return []
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
lib/tool_shed/galaxy_install/repository_util.py
+2 -5
View File
@@ -19,6 +19,7 @@ from tool_shed.util import repository_dependency_util
from tool_shed.util import metadata_util
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
from xml.etree import ElementTree as XmlET
from galaxy import eggs
import pkg_resources
@@ -28,10 +29,6 @@ from mercurial import commands
from mercurial import hg
from mercurial import ui
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree
from elementtree.ElementTree import Element
log = logging.getLogger( __name__ )
def create_repo_info_dict( trans, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_name=None, repository=None,
@@ -401,7 +398,7 @@ def initiate_repository_installation( trans, installation_dict ):
else:
# Appending a new section to trans.app.toolbox.tool_panel
log.debug( "Loading new tool panel section: %s" % new_tool_panel_section )
elem = Element( 'section' )
elem = XmlET.Element( 'section' )
elem.attrib[ 'name' ] = new_tool_panel_section
elem.attrib[ 'id' ] = section_id
elem.attrib[ 'version' ] = ''
lib/tool_shed/galaxy_install/tool_dependencies/install_util.py
+1 -10
View File
@@ -14,15 +14,6 @@ from tool_shed.util import xml_util
from galaxy.model.orm import and_
from galaxy.web import url_for
from galaxy import eggs
import pkg_resources
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree
from elementtree import ElementInclude
from elementtree.ElementTree import Element
from elementtree.ElementTree import SubElement
log = logging.getLogger( __name__ )
def clean_tool_shed_url( base_url ):
@@ -479,7 +470,7 @@ def populate_actions_dict( app, dependent_install_dir, required_install_dir, too
tool_dependency = None
action_dict = {}
if tool_dependencies_config:
required_td_tree = xml_util.parse_xml( tool_dependencies_config )
required_td_tree, error_message = xml_util.parse_xml( tool_dependencies_config )
if required_td_tree:
required_td_root = required_td_tree.getroot()
for required_td_elem in required_td_root:
lib/tool_shed/tool_shed_registry.py
+23 -20
View File
@@ -10,29 +10,32 @@ log = logging.getLogger( __name__ )
class Registry( object ):
def __init__( self, root_dir=None, config=None ):
self.tool_sheds = odict()
self.tool_sheds = odict()
self.tool_sheds_auth = odict()
if root_dir and config:
# Parse tool_sheds_conf.xml
tree = xml_util.parse_xml( config )
root = tree.getroot()
log.debug( 'Loading references to tool sheds from %s' % config )
for elem in root.findall( 'tool_shed' ):
try:
name = elem.get( 'name', None )
url = elem.get( 'url', None )
username = elem.get( 'user', None )
password = elem.get( 'pass', None )
if name and url:
self.tool_sheds[ name ] = url
self.tool_sheds_auth[ name ] = None
log.debug( 'Loaded reference to tool shed: %s' % name )
if name and url and username and password:
pass_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
pass_mgr.add_password(None, url, username, password)
self.tool_sheds_auth[ name ] = pass_mgr
except Exception, e:
log.warning( 'Error loading reference to tool shed "%s", problem: %s' % ( name, str( e ) ) )
tree, error_message = xml_util.parse_xml( config )
if tree is None:
log.warning( "Unable to load references to tool sheds defined in file %s" % str( config ) )
else:
root = tree.getroot()
log.debug( 'Loading references to tool sheds from %s' % config )
for elem in root.findall( 'tool_shed' ):
try:
name = elem.get( 'name', None )
url = elem.get( 'url', None )
username = elem.get( 'user', None )
password = elem.get( 'pass', None )
if name and url:
self.tool_sheds[ name ] = url
self.tool_sheds_auth[ name ] = None
log.debug( 'Loaded reference to tool shed: %s' % name )
if name and url and username and password:
pass_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
pass_mgr.add_password( None, url, username, password )
self.tool_sheds_auth[ name ] = pass_mgr
except Exception, e:
log.warning( 'Error loading reference to tool shed "%s", problem: %s' % ( name, str( e ) ) )
def password_manager_for_url( self, url ):
"""
lib/tool_shed/util/commit_util.py
+8 -14
View File
@@ -185,14 +185,11 @@ def handle_gzip( repository, uploaded_file_name ):
def handle_repository_dependencies_definition( trans, repository_dependencies_config ):
altered = False
try:
# Make sure we're looking at a valid repository_dependencies.xml file.
tree = xml_util.parse_xml( repository_dependencies_config )
root = tree.getroot()
except Exception, e:
error_message = "Error parsing %s in handle_repository_dependencies_definition: " % str( repository_dependencies_config )
log.exception( error_message )
# Make sure we're looking at a valid repository_dependencies.xml file.
tree, error_message = xml_util.parse_xml( repository_dependencies_config )
if tree is None:
return False, None
root = tree.getroot()
if root.tag == 'repositories':
for index, elem in enumerate( root ):
# <repository name="molecule_datatypes" owner="test" changeset_revision="1a070566e9c6" />
@@ -232,14 +229,11 @@ def handle_repository_dependency_elem( trans, elem ):
def handle_tool_dependencies_definition( trans, tool_dependencies_config ):
altered = False
try:
# Make sure we're looking at a valid tool_dependencies.xml file.
tree = xml_util.parse_xml( tool_dependencies_config )
root = tree.getroot()
except Exception, e:
error_message = "Error parsing %s in handle_tool_dependencies_definition: " % str( tool_dependencies_config )
log.exception( error_message )
# Make sure we're looking at a valid tool_dependencies.xml file.
tree, error_message = xml_util.parse_xml( tool_dependencies_config )
if tree is None:
return False, None
root = tree.getroot()
if root.tag == 'tool_dependency':
for root_index, root_elem in enumerate( root ):
# <package name="eigen" version="2.0.17">
lib/tool_shed/util/common_install_util.py
+3 -13
View File
@@ -13,17 +13,10 @@ from tool_shed.util import encoding_util
from tool_shed.util import data_manager_util
from tool_shed.util import datatype_util
from tool_shed.util import tool_util
from tool_shed.util import xml_util
from tool_shed.galaxy_install.tool_dependencies.install_util import install_package
from tool_shed.galaxy_install.tool_dependencies.install_util import set_environment
import pkg_resources
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree
from elementtree import ElementInclude
from elementtree.ElementTree import Element
from elementtree.ElementTree import SubElement
log = logging.getLogger( __name__ )
def activate_repository( trans, repository ):
@@ -318,13 +311,10 @@ def handle_tool_dependencies( app, tool_shed_repository, tool_dependencies_confi
sa_session = app.model.context.current
installed_tool_dependencies = []
# Parse the tool_dependencies.xml config.
try:
tree = ElementTree.parse( tool_dependencies_config )
except Exception, e:
log.debug( "Exception attempting to parse %s: %s" % ( str( tool_dependencies_config ), str( e ) ) )
tree, error_message = xml_util.parse_xml( tool_dependencies_config )
if tree is None:
return installed_tool_dependencies
root = tree.getroot()
ElementInclude.include( root )
fabric_version_checked = False
for elem in root:
if elem.tag == 'package':
lib/tool_shed/util/common_util.py
+16 -11
View File
@@ -11,7 +11,9 @@ def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_scri
tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) )
# Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
migrated_tool_configs_dict = odict()
tree = xml_util.parse_xml( tools_xml_file_path )
tree, error_message = xml_util.parse_xml( tools_xml_file_path )
if tree is None:
return False, odict()
root = tree.getroot()
tool_shed = root.get( 'name' )
tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed )
@@ -48,15 +50,16 @@ def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_scri
# Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names.
missing_tool_configs_dict = odict()
for tool_panel_config in tool_panel_configs:
tree = xml_util.parse_xml( tool_panel_config )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict )
elif elem.tag == 'section':
for section_elem in elem:
if section_elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict )
tree, error_message = xml_util.parse_xml( tool_panel_config )
if tree:
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict )
elif elem.tag == 'section':
for section_elem in elem:
if section_elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict )
else:
exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % ( tool_shed, app.config.tool_sheds_config )
exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n'
@@ -78,7 +81,9 @@ def get_non_shed_tool_panel_configs( app ):
for config_filename in app.config.tool_configs:
# Any config file that includes a tool_path attribute in the root tag set like the following is shed-related.
# <toolbox tool_path="../shed_tools">
tree = xml_util.parse_xml( config_filename )
tree, error_message = xml_util.parse_xml( config_filename )
if tree is None:
continue
root = tree.getroot()
tool_path = root.get( 'tool_path', None )
if tool_path is None:
lib/tool_shed/util/data_manager_util.py
+42 -36
View File
@@ -21,7 +21,10 @@ def install_data_managers( app, shed_data_manager_conf_filename, metadata_dict,
for tool_tup in repository_tools_tups:
repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] )
# Load existing data managers.
config_elems = [ elem for elem in xml_util.parse_xml( shed_data_manager_conf_filename ).getroot() ]
tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename )
if tree is None:
return rval
config_elems = [ elem for elem in tree.getroot() ]
repo_data_manager_conf_filename = metadata_dict['data_manager'].get( 'config_filename', None )
if repo_data_manager_conf_filename is None:
log.debug( "No data_manager_conf.xml file has been defined." )
@@ -29,7 +32,9 @@ def install_data_managers( app, shed_data_manager_conf_filename, metadata_dict,
data_manager_config_has_changes = False
relative_repo_data_manager_dir = os.path.join( shed_config_dict.get( 'tool_path', '' ), relative_install_dir )
repo_data_manager_conf_filename = os.path.join( relative_repo_data_manager_dir, repo_data_manager_conf_filename )
tree = xml_util.parse_xml( repo_data_manager_conf_filename )
tree, error_message = xml_util.parse_xml( repo_data_manager_conf_filename )
if tree is None:
return rval
root = tree.getroot()
for elem in root:
if elem.tag == 'data_manager':
@@ -89,37 +94,38 @@ def remove_from_data_manager( app, repository ):
metadata_dict = repository.metadata
if metadata_dict and 'data_manager' in metadata_dict:
shed_data_manager_conf_filename = app.config.shed_data_manager_config_file
tree = xml_util.parse_xml( shed_data_manager_conf_filename )
root = tree.getroot()
assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename )
guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ]
load_old_data_managers_by_guid = {}
data_manager_config_has_changes = False
config_elems = []
for elem in root:
# Match Data Manager elements by guid and installed_changeset_revision
elem_matches_removed_data_manager = False
if elem.tag == 'data_manager':
guid = elem.get( 'guid', None )
if guid in guids:
tool_elem = elem.find( 'tool' )
if tool_elem is not None:
installed_changeset_revision_elem = tool_elem.find( 'installed_changeset_revision' )
if installed_changeset_revision_elem is not None:
if installed_changeset_revision_elem.text == repository.installed_changeset_revision:
elem_matches_removed_data_manager = True
else:
# This is a different version, which had been previously overridden
load_old_data_managers_by_guid[ guid ] = elem
if elem_matches_removed_data_manager:
data_manager_config_has_changes = True
else:
config_elems.append( elem )
# Remove data managers from in memory
app.data_managers.remove_manager( guids )
# Load other versions of any now uninstalled data managers, if any
for elem in load_old_data_managers_by_guid.itervalues():
app.data_managers.load_manager_from_elem( elem )
# Persist the altered shed_data_manager_config file.
if data_manager_config_has_changes:
data_manager_config_elems_to_xml_file( app, config_elems, shed_data_manager_conf_filename )
tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename )
if tree:
root = tree.getroot()
assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename )
guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ]
load_old_data_managers_by_guid = {}
data_manager_config_has_changes = False
config_elems = []
for elem in root:
# Match Data Manager elements by guid and installed_changeset_revision
elem_matches_removed_data_manager = False
if elem.tag == 'data_manager':
guid = elem.get( 'guid', None )
if guid in guids:
tool_elem = elem.find( 'tool' )
if tool_elem is not None:
installed_changeset_revision_elem = tool_elem.find( 'installed_changeset_revision' )
if installed_changeset_revision_elem is not None:
if installed_changeset_revision_elem.text == repository.installed_changeset_revision:
elem_matches_removed_data_manager = True
else:
# This is a different version, which had been previously overridden
load_old_data_managers_by_guid[ guid ] = elem
if elem_matches_removed_data_manager:
data_manager_config_has_changes = True
else:
config_elems.append( elem )
# Remove data managers from in memory
app.data_managers.remove_manager( guids )
# Load other versions of any now uninstalled data managers, if any
for elem in load_old_data_managers_by_guid.itervalues():
app.data_managers.load_manager_from_elem( elem )
# Persist the altered shed_data_manager_config file.
if data_manager_config_has_changes:
data_manager_config_elems_to_xml_file( app, config_elems, shed_data_manager_conf_filename )
lib/tool_shed/util/datatype_util.py
+2 -12
View File
@@ -5,14 +5,6 @@ from galaxy import eggs
from tool_shed.util import xml_util
import tool_shed.util.shed_util_common as suc
import pkg_resources
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree
from elementtree import ElementInclude
from elementtree.ElementTree import Element
from elementtree.ElementTree import SubElement
log = logging.getLogger( __name__ )
def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_install_dir, deactivate=False, override=True ):
@@ -23,10 +15,8 @@ def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_
be False when a tool shed repository is being installed. Since installation is occurring after the datatypes registry
has been initialized, the registry's contents cannot be overridden by conflicting data types.
"""
try:
tree = xml_util.parse_xml( datatypes_config )
except Exception, e:
log.debug( "Error parsing %s, exception: %s" % ( datatypes_config, str( e ) ) )
tree, error_message = xml_util.parse_xml( datatypes_config )
if tree is None:
return None, None
datatypes_config_root = tree.getroot()
registration = datatypes_config_root.find( 'registration' )
lib/tool_shed/util/metadata_util.py
+16 -33
View File
@@ -26,10 +26,6 @@ from mercurial import commands
from mercurial import hg
from mercurial import ui
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree
from elementtree import ElementInclude
log = logging.getLogger( __name__ )
REPOSITORY_DATA_MANAGER_CONFIG_FILENAME = "data_manager_conf.xml"
@@ -363,12 +359,9 @@ def generate_data_manager_metadata( app, repository, repo_dir, data_manager_conf
'invalid_data_managers': invalid_data_managers,
'error_messages': [] }
metadata_dict[ 'data_manager' ] = data_manager_metadata
try:
tree = xml_util.parse_xml( data_manager_config_filename )
except Exception, e:
tree, error_message = xml_util.parse_xml( data_manager_config_filename )
if tree is None:
# We are not able to load any data managers.
error_message = 'There was an error parsing your Data Manager config file "%s": %s' % ( data_manager_config_filename, e )
log.error( error_message )
data_manager_metadata[ 'error_messages' ].append( error_message )
return metadata_dict
tool_path = None
@@ -436,13 +429,10 @@ def generate_data_manager_metadata( app, repository, repo_dir, data_manager_conf
def generate_datatypes_metadata( app, repository, repository_clone_url, repository_files_dir, datatypes_config, metadata_dict ):
"""Update the received metadata_dict with information from the parsed datatypes_config."""
try:
tree = ElementTree.parse( datatypes_config )
except Exception, e:
log.debug( "Exception attempting to parse %s: %s" % ( str( datatypes_config ), str( e ) ) )
tree, error_message = xml_util.parse_xml( datatypes_config )
if tree is None:
return metadata_dict
root = tree.getroot()
ElementInclude.include( root )
repository_datatype_code_files = []
datatype_files = root.find( 'datatype_files' )
if datatype_files:
@@ -637,14 +627,13 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio
if os.path.getsize( full_path ) > 0:
if not ( checkers.check_binary( full_path ) or checkers.check_image( full_path ) or checkers.check_gzip( full_path )[ 0 ]
or checkers.check_bz2( full_path )[ 0 ] or checkers.check_zip( full_path ) ):
try:
# Make sure we're looking at a tool config and not a display application config or something else.
element_tree = xml_util.parse_xml( full_path )
# Make sure we're looking at a tool config and not a display application config or something else.
element_tree, error_message = xml_util.parse_xml( full_path )
if element_tree is None:
is_tool = False
else:
element_tree_root = element_tree.getroot()
is_tool = element_tree_root.tag == 'tool'
except Exception, e:
log.debug( "Error parsing %s, exception: %s" % ( full_path, str( e ) ) )
is_tool = False
if is_tool:
tool, valid, error_message = tool_util.load_tool_from_config( app, app.security.encode_id( repository.id ), full_path )
if tool is None:
@@ -750,15 +739,13 @@ def generate_repository_dependency_metadata( app, repository_dependencies_config
is called from the tool shed as well as from Galaxy.
"""
error_message = ''
try:
# Make sure we're looking at a valid repository_dependencies.xml file.
tree = xml_util.parse_xml( repository_dependencies_config )
# Make sure we're looking at a valid repository_dependencies.xml file.
tree, error_message = xml_util.parse_xml( repository_dependencies_config )
if tree is None:
xml_is_valid = False
else:
root = tree.getroot()
xml_is_valid = root.tag == 'repositories'
except Exception, e:
error_message = "Error parsing %s, exception: %s" % ( repository_dependencies_config, str( e ) )
log.debug( error_message )
xml_is_valid = False
if xml_is_valid:
invalid_repository_dependencies_dict = dict( description=root.get( 'description' ) )
invalid_repository_dependency_tups = []
@@ -795,14 +782,10 @@ def generate_tool_dependency_metadata( app, repository, changeset_revision, repo
else:
original_valid_tool_dependencies_dict = None
original_invalid_tool_dependencies_dict = None
try:
tree = ElementTree.parse( tool_dependencies_config )
except Exception, e:
error_message = "Exception attempting to parse tool_dependencies.xml: %s" %str( e )
log.debug( error_message )
tree, error_message = xml_util.parse_xml( tool_dependencies_config )
if tree is None:
return metadata_dict, error_message
root = tree.getroot()
ElementInclude.include( root )
tool_dependency_is_valid = True
valid_tool_dependencies_dict = {}
invalid_tool_dependencies_dict = {}
lib/tool_shed/util/shed_util_common.py
+26 -28
View File
@@ -15,18 +15,13 @@ from galaxy.model.orm import and_
import sqlalchemy.orm.exc
from tool_shed.util import common_util
from tool_shed.util import xml_util
from xml.etree import ElementTree as XmlET
from galaxy import eggs
import pkg_resources
pkg_resources.require( 'mercurial' )
from mercurial import hg, ui, commands
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree
from elementtree import ElementInclude
from elementtree.ElementTree import Element
from elementtree.ElementTree import SubElement
eggs.require( 'markupsafe' )
import markupsafe
@@ -290,22 +285,22 @@ def generate_sharable_link_for_repository_in_tool_shed( trans, repository, chang
def generate_tool_elem( tool_shed, repository_name, changeset_revision, owner, tool_file_path, tool, tool_section ):
"""Create and return an ElementTree tool Element."""
if tool_section is not None:
tool_elem = SubElement( tool_section, 'tool' )
tool_elem = XmlET.SubElement( tool_section, 'tool' )
else:
tool_elem = Element( 'tool' )
tool_elem = XmlET.Element( 'tool' )
tool_elem.attrib[ 'file' ] = tool_file_path
tool_elem.attrib[ 'guid' ] = tool.guid
tool_shed_elem = SubElement( tool_elem, 'tool_shed' )
tool_shed_elem = XmlET.SubElement( tool_elem, 'tool_shed' )
tool_shed_elem.text = tool_shed
repository_name_elem = SubElement( tool_elem, 'repository_name' )
repository_name_elem = XmlET.SubElement( tool_elem, 'repository_name' )
repository_name_elem.text = repository_name
repository_owner_elem = SubElement( tool_elem, 'repository_owner' )
repository_owner_elem = XmlET.SubElement( tool_elem, 'repository_owner' )
repository_owner_elem.text = owner
changeset_revision_elem = SubElement( tool_elem, 'installed_changeset_revision' )
changeset_revision_elem = XmlET.SubElement( tool_elem, 'installed_changeset_revision' )
changeset_revision_elem.text = changeset_revision
id_elem = SubElement( tool_elem, 'id' )
id_elem = XmlET.SubElement( tool_elem, 'id' )
id_elem.text = tool.id
version_elem = SubElement( tool_elem, 'version' )
version_elem = XmlET.SubElement( tool_elem, 'version' )
version_elem.text = tool.version
return tool_elem
@@ -335,7 +330,9 @@ def generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository ):
file_name = strip_path( tool_config )
guids_and_configs[ guid ] = file_name
# Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict.
tree = xml_util.parse_xml( shed_tool_conf )
tree, error_message = xml_util.parse_xml( shed_tool_conf )
if tree is None:
return tool_panel_dict
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
@@ -1253,20 +1250,21 @@ def update_in_shed_tool_config( app, repository ):
for tool_config_filename, guid, tool in repository_tools_tups:
guid_to_tool_elem_dict[ guid ] = generate_tool_elem( tool_shed, repository.name, repository.changeset_revision, repository.owner or '', tool_config_filename, tool, None )
config_elems = []
tree = xml_util.parse_xml( shed_tool_conf )
root = tree.getroot()
for elem in root:
if elem.tag == 'section':
for i, tool_elem in enumerate( elem ):
guid = tool_elem.attrib.get( 'guid' )
tree, error_message = xml_util.parse_xml( shed_tool_conf )
if tree:
root = tree.getroot()
for elem in root:
if elem.tag == 'section':
for i, tool_elem in enumerate( elem ):
guid = tool_elem.attrib.get( 'guid' )
if guid in guid_to_tool_elem_dict:
elem[i] = guid_to_tool_elem_dict[ guid ]
elif elem.tag == 'tool':
guid = elem.attrib.get( 'guid' )
if guid in guid_to_tool_elem_dict:
elem[i] = guid_to_tool_elem_dict[ guid ]
elif elem.tag == 'tool':
guid = elem.attrib.get( 'guid' )
if guid in guid_to_tool_elem_dict:
elem = guid_to_tool_elem_dict[ guid ]
config_elems.append( elem )
config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path )
elem = guid_to_tool_elem_dict[ guid ]
config_elems.append( elem )
config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path )
def update_repository( repo, ctx_rev=None ):
"""
lib/tool_shed/util/tool_dependency_util.py
+3 -9
View File
@@ -5,10 +5,7 @@ from galaxy import eggs
from galaxy import util
from galaxy.model.orm import and_
import tool_shed.util.shed_util_common as suc
eggs.require( 'elementtree' )
from elementtree import ElementTree
from elementtree import ElementInclude
from tool_shed.util import xml_util
log = logging.getLogger( __name__ )
@@ -67,13 +64,10 @@ def create_tool_dependency_objects( app, tool_shed_repository, relative_install_
relative_install_dir = os.path.join( shed_config_dict.get( 'tool_path' ), relative_install_dir )
# Get the tool_dependencies.xml file from the repository.
tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', relative_install_dir )
try:
tree = ElementTree.parse( tool_dependencies_config )
except Exception, e:
log.debug( "Exception attempting to parse tool_dependencies.xml: %s" %str( e ) )
tree, error_message = xml_util.parse_xml( tool_dependencies_config )
if tree is None:
return tool_dependency_objects
root = tree.getroot()
ElementInclude.include( root )
fabric_version_checked = False
for elem in root:
tool_dependency_type = elem.tag
lib/tool_shed/util/tool_util.py
+42 -43
View File
@@ -14,6 +14,7 @@ from galaxy.tools.search import ToolBoxSearch
from galaxy.web.form_builder import SelectField
from tool_shed.util import xml_util
import tool_shed.util.shed_util_common as suc
from xml.etree import ElementTree as XmlET
import pkg_resources
@@ -22,12 +23,6 @@ from mercurial import commands
from mercurial import hg
from mercurial import ui
pkg_resources.require( 'elementtree' )
from elementtree import ElementTree
from elementtree import ElementInclude
from elementtree.ElementTree import Element
from elementtree.ElementTree import SubElement
log = logging.getLogger( __name__ )
def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ):
@@ -37,15 +32,16 @@ def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ):
shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
tool_path = shed_tool_conf_dict[ 'tool_path' ]
config_elems = []
tree = xml_util.parse_xml( shed_tool_conf )
root = tree.getroot()
for elem in root:
config_elems.append( elem )
# Add the elements to the in-memory list of config_elems.
for elem_entry in elem_list:
config_elems.append( elem_entry )
# Persist the altered shed_tool_config file.
suc.config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path )
tree, error_message = xml_util.parse_xml( shed_tool_conf )
if tree:
root = tree.getroot()
for elem in root:
config_elems.append( elem )
# Add the elements to the in-memory list of config_elems.
for elem_entry in elem_list:
config_elems.append( elem_entry )
# Persist the altered shed_tool_config file.
suc.config_elems_to_xml_file( app, config_elems, shed_tool_conf, tool_path )
def add_to_tool_panel( app, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner, shed_tool_conf, tool_panel_dict,
new_install=True ):
@@ -381,7 +377,7 @@ def generate_tool_section_element_from_dict( tool_section_dict ):
# { id: <ToolSection id>, version : <ToolSection version>, name : <TooSection name>}
if tool_section_dict[ 'id' ]:
# Create a new tool section.
tool_section = Element( 'section' )
tool_section = XmlET.Element( 'section' )
tool_section.attrib[ 'id' ] = tool_section_dict[ 'id' ]
tool_section.attrib[ 'name' ] = tool_section_dict[ 'name' ]
tool_section.attrib[ 'version' ] = tool_section_dict[ 'version' ]
@@ -639,7 +635,7 @@ def handle_tool_panel_selection( trans, metadata, no_changes_checked, tool_panel
tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ]
else:
# The section in which the tool was originally loaded used to be in the tool panel, but no longer is.
elem = Element( 'section' )
elem = XmlET.Element( 'section' )
elem.attrib[ 'name' ] = original_section_name
elem.attrib[ 'id' ] = original_section_id
elem.attrib[ 'version' ] = ''
@@ -657,7 +653,7 @@ def handle_tool_panel_selection( trans, metadata, no_changes_checked, tool_panel
else:
# Appending a new section to trans.app.toolbox.tool_panel
log.debug( "Loading new tool panel section: %s" % new_tool_panel_section )
elem = Element( 'section' )
elem = XmlET.Element( 'section' )
elem.attrib[ 'name' ] = new_tool_panel_section
elem.attrib[ 'id' ] = section_id
elem.attrib[ 'version' ] = ''
@@ -804,7 +800,9 @@ def load_tool_from_tmp_config( trans, repo, repository_id, ctx, ctx_file, work_d
message = ''
tmp_tool_config = suc.get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir )
if tmp_tool_config:
element_tree = xml_util.parse_xml( tmp_tool_config )
element_tree, error_message = xml_util.parse_xml( tmp_tool_config )
if element_tree is None:
return tool, message
element_tree_root = element_tree.getroot()
# Look for code files required by the tool config.
tmp_code_files = []
@@ -846,30 +844,31 @@ def remove_from_shed_tool_config( trans, shed_tool_conf_dict, guids_to_remove ):
shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
tool_path = shed_tool_conf_dict[ 'tool_path' ]
config_elems = []
tree = xml_util.parse_xml( shed_tool_conf )
root = tree.getroot()
for elem in root:
config_elems.append( elem )
config_elems_to_remove = []
for config_elem in config_elems:
if config_elem.tag == 'section':
tool_elems_to_remove = []
for tool_elem in config_elem:
if tool_elem.get( 'guid' ) in guids_to_remove:
tool_elems_to_remove.append( tool_elem )
for tool_elem in tool_elems_to_remove:
# Remove all of the appropriate tool sub-elements from the section element.
config_elem.remove( tool_elem )
if len( config_elem ) < 1:
# Keep a list of all empty section elements so they can be removed.
config_elems_to_remove.append( config_elem )
elif config_elem.tag == 'tool':
if config_elem.get( 'guid' ) in guids_to_remove:
config_elems_to_remove.append( config_elem )
for config_elem in config_elems_to_remove:
config_elems.remove( config_elem )
# Persist the altered in-memory version of the tool config.
suc.config_elems_to_xml_file( trans.app, config_elems, shed_tool_conf, tool_path )
tree, error_message = xml_util.parse_xml( shed_tool_conf )
if tree:
root = tree.getroot()
for elem in root:
config_elems.append( elem )
config_elems_to_remove = []
for config_elem in config_elems:
if config_elem.tag == 'section':
tool_elems_to_remove = []
for tool_elem in config_elem:
if tool_elem.get( 'guid' ) in guids_to_remove:
tool_elems_to_remove.append( tool_elem )
for tool_elem in tool_elems_to_remove:
# Remove all of the appropriate tool sub-elements from the section element.
config_elem.remove( tool_elem )
if len( config_elem ) < 1:
# Keep a list of all empty section elements so they can be removed.
config_elems_to_remove.append( config_elem )
elif config_elem.tag == 'tool':
if config_elem.get( 'guid' ) in guids_to_remove:
config_elems_to_remove.append( config_elem )
for config_elem in config_elems_to_remove:
config_elems.remove( config_elem )
# Persist the altered in-memory version of the tool config.
suc.config_elems_to_xml_file( trans.app, config_elems, shed_tool_conf, tool_path )
def remove_from_tool_panel( trans, repository, shed_tool_conf, uninstall ):
"""A tool shed repository is being deactivated or uninstalled so handle tool panel alterations accordingly."""
lib/tool_shed/util/xml_util.py
+39 -10
View File
@@ -1,13 +1,17 @@
import logging
import os
import sys
import tempfile
from xml.etree import ElementTree as XmlET
import xml.etree.ElementTree
log = logging.getLogger( __name__ )
using_python_27 = sys.version_info[ :2 ] >= ( 2, 7 )
class CommentedTreeBuilder ( XmlET.XMLTreeBuilder ):
class Py26CommentedTreeBuilder ( XmlET.XMLTreeBuilder ):
# Python 2.6 uses ElementTree 1.2.x.
def __init__ ( self, html=0, target=None ):
XmlET.XMLTreeBuilder.__init__( self, html, target )
self._parser.CommentHandler = self.handle_comment
@@ -17,6 +21,15 @@ class CommentedTreeBuilder ( XmlET.XMLTreeBuilder ):
self._target.data( data )
self._target.end( XmlET.Comment )
class Py27CommentedTreeBuilder ( XmlET.TreeBuilder ):
# Python 2.7 uses ElementTree 1.3.x.
def comment( self, data ):
self.start( XmlET.Comment, {} )
self.data( data )
self.end( XmlET.Comment )
def create_and_write_tmp_file( elem ):
tmp_str = xml_to_string( elem )
fh = tempfile.NamedTemporaryFile( 'wb' )
@@ -30,14 +43,30 @@ def create_and_write_tmp_file( elem ):
def parse_xml( file_name ):
"""Returns a parsed xml tree with comments intact."""
try:
fobj = open( file_name, 'r' )
tree = XmlET.parse( fobj, parser=CommentedTreeBuilder() )
fobj.close()
except Exception, e:
log.exception( "Exception attempting to parse %s: %s" % ( str( file_name ), str( e ) ) )
return None
return tree
error_message = ''
fobj = open( file_name, 'r' )
if using_python_27:
try:
tree = XmlET.parse( fobj, parser=XmlET.XMLParser( target=Py27CommentedTreeBuilder() ) )
except Exception, e:
fobj.close()
error_message = "Exception attempting to parse %s: %s" % ( str( file_name ), str( e ) )
log.exception( error_message )
return None, error_message
else:
try:
tree = XmlET.parse( fobj, parser=Py26CommentedTreeBuilder() )
except Exception, e:
fobj.close()
error_message = "Exception attempting to parse %s: %s" % ( str( file_name ), str( e ) )
log.exception( error_message )
return None, error_message
fobj.close()
return tree, error_message
def xml_to_string( elem, encoding='utf-8' ):
return '%s\n' % xml.etree.ElementTree.tostring( elem, encoding=encoding )
if using_python_27:
xml_str = '%s\n' % xml.etree.ElementTree.tostring( elem, encoding=encoding, method="xml" )
else:
xml_str = '%s\n' % xml.etree.ElementTree.tostring( elem, encoding=encoding )
return xml_str
test/tool_shed/functional/test_0010_repository_with_tool_dependencies.py
+1 -1
View File
@@ -118,7 +118,7 @@ class TestFreebayesRepository( ShedTwillTestCase ):
uncompress_file=False,
remove_repo_files_not_in_tar=False,
commit_message='Uploaded malformed tool dependency XML.',
strings_displayed=[ 'Exception attempting to parse tool_dependencies.xml', 'not well-formed' ],
strings_displayed=[ 'Exception attempting to parse', 'not well-formed' ],
strings_not_displayed=[] )
def test_0030_upload_invalid_tool_dependency_xml( self ):
test/tool_shed/functional/test_1010_install_repository_with_tool_dependencies.py
+1 -1
View File
@@ -71,7 +71,7 @@ class ToolWithToolDependencies( ShedTwillTestCase ):
uncompress_file=False,
remove_repo_files_not_in_tar=False,
commit_message='Uploaded malformed tool dependency XML.',
strings_displayed=[ 'Exception attempting to parse tool_dependencies.xml', 'not well-formed' ],
strings_displayed=[ 'Exception attempting to parse', 'not well-formed' ],
strings_not_displayed=[] )
self.upload_file( repository,
filename=os.path.join( 'freebayes', 'invalid_tool_dependencies', 'tool_dependencies.xml' ),
test/tool_shed/functional/test_1210_uninstall_reinstall_repository_with_tool_dependencies.py
+1 -1
View File
@@ -65,7 +65,7 @@ class UninstallingAndReinstallingRepositories( ShedTwillTestCase ):
uncompress_file=False,
remove_repo_files_not_in_tar=False,
commit_message='Uploaded malformed tool dependency XML.',
strings_displayed=[ 'Exception attempting to parse tool_dependencies.xml', 'not well-formed' ],
strings_displayed=[ 'Exception attempting to parse', 'not well-formed' ],
strings_not_displayed=[] )
self.upload_file( repository,
filename=os.path.join( 'freebayes', 'invalid_tool_dependencies', 'tool_dependencies.xml' ),