Merged from next-stable

This commit is contained in:
Greg Von Kuster
2013-05-23 14:58:00 -04:00
12 changed files with 89 additions and 82 deletions
@@ -14,6 +14,7 @@ from tool_shed.util import metadata_util
from tool_shed.util import repository_dependency_util
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
from tool_shed.util import xml_util
from galaxy import eggs
eggs.require( 'mercurial' )
@@ -131,7 +132,7 @@ class UploadController( BaseUIController ):
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name )
if altered:
tmp_filename = commit_util.create_and_write_tmp_file( root_elem )
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, full_path )
else:
shutil.move( uploaded_file_name, full_path )
@@ -140,7 +141,7 @@ class UploadController( BaseUIController ):
# are missing and if so, set them appropriately.
altered, root_elem = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
if altered:
tmp_filename = commit_util.create_and_write_tmp_file( root_elem )
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, full_path )
else:
shutil.move( uploaded_file_name, full_path )
@@ -268,13 +269,13 @@ class UploadController( BaseUIController ):
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name )
if altered:
tmp_filename = commit_util.create_and_write_tmp_file( root_elem )
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, uploaded_file_name )
elif os.path.split( uploaded_file_name )[ -1 ] == 'tool_dependencies.xml':
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
if altered:
tmp_filename = commit_util.create_and_write_tmp_file( root_elem )
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, uploaded_file_name )
if ok:
repo_path = os.path.join( full_path, relative_path )
@@ -330,13 +331,13 @@ class UploadController( BaseUIController ):
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem = commit_util.handle_repository_dependencies_definition( trans, uploaded_file_name )
if altered:
tmp_filename = commit_util.create_and_write_tmp_file( root_elem )
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, uploaded_file_name )
elif os.path.split( uploaded_file_name )[ -1 ] == 'tool_dependencies.xml':
# Inspect the contents of the file to see if changeset_revision values are missing and if so, set them appropriately.
altered, root_elem = commit_util.handle_tool_dependencies_definition( trans, uploaded_file_name )
if altered:
tmp_filename = commit_util.create_and_write_tmp_file( root_elem )
tmp_filename = xml_util.create_and_write_tmp_file( root_elem )
shutil.move( tmp_filename, uploaded_file_name )
return commit_util.handle_directory_changes( trans,
repository,
@@ -15,6 +15,7 @@ from tool_shed.util import datatype_util
from tool_shed.util import metadata_util
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
from tool_shed.util import xml_util
from galaxy.util.odict import odict
@@ -37,13 +38,13 @@ class InstallManager( object ):
self.proprietary_tool_confs = self.non_shed_tool_panel_configs
self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number )
# Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config.
tree = util.parse_xml( migrated_tools_config )
tree = xml_util.parse_xml( migrated_tools_config )
root = tree.getroot()
self.tool_path = root.get( 'tool_path' )
print "Repositories will be installed into configured tool_path location ", str( self.tool_path )
# Parse tool_shed_install_config to check each of the tools.
self.tool_shed_install_config = tool_shed_install_config
tree = util.parse_xml( tool_shed_install_config )
tree = xml_util.parse_xml( tool_shed_install_config )
root = tree.getroot()
self.tool_shed = suc.clean_tool_shed_url( root.get( 'name' ) )
self.repository_owner = common_util.REPOSITORY_OWNER
@@ -107,7 +108,7 @@ class InstallManager( object ):
tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) )
# Parse the XML and load the file attributes for later checking against the integrated elements from self.proprietary_tool_confs.
migrated_tool_configs = []
tree = util.parse_xml( tools_xml_file_path )
tree = xml_util.parse_xml( tools_xml_file_path )
root = tree.getroot()
for elem in root:
if elem.tag == 'repository':
@@ -116,7 +117,7 @@ class InstallManager( object ):
# Parse each file in self.proprietary_tool_confs and generate the integrated list of tool panel Elements that contain them.
tool_panel_elems = []
for proprietary_tool_conf in self.proprietary_tool_confs:
tree = util.parse_xml( proprietary_tool_conf )
tree = xml_util.parse_xml( proprietary_tool_conf )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
@@ -10,6 +10,7 @@ import tool_shed.util.shed_util_common as suc
import tool_shed.util.common_util as cu
from tool_shed.util import encoding_util
from tool_shed.util import tool_dependency_util
from tool_shed.util import xml_util
from galaxy.model.orm import and_
from galaxy.web import url_for
@@ -478,7 +479,7 @@ def populate_actions_dict( app, dependent_install_dir, required_install_dir, too
tool_dependency = None
action_dict = {}
if tool_dependencies_config:
required_td_tree = parse_xml( tool_dependencies_config )
required_td_tree = xml_util.parse_xml( tool_dependencies_config )
if required_td_tree:
required_td_root = required_td_tree.getroot()
for required_td_elem in required_td_root:
@@ -623,17 +624,6 @@ def strip_path( fpath ):
file_name = fpath
return file_name
def parse_xml( file_name ):
"""Returns a parsed xml tree."""
try:
tree = ElementTree.parse( file_name )
except Exception, e:
print "Exception attempting to parse ", file_name, ": ", str( e )
return None
root = tree.getroot()
ElementInclude.include( root )
return tree
def url_join( *args ):
parts = []
for arg in args:
+2 -3
View File
@@ -1,9 +1,8 @@
import logging
import sys
import urllib2
from galaxy.util import parse_xml
from galaxy.util.odict import odict
from xml.etree import ElementTree
from tool_shed.util import xml_util
log = logging.getLogger( __name__ )
@@ -15,7 +14,7 @@ class Registry( object ):
self.tool_sheds_auth = odict()
if root_dir and config:
# Parse tool_sheds_conf.xml
tree = parse_xml( config )
tree = xml_util.parse_xml( config )
root = tree.getroot()
log.debug( 'Loading references to tool sheds from %s' % config )
for elem in root.findall( 'tool_shed' ):
+3 -14
View File
@@ -9,7 +9,7 @@ from galaxy.util import json
from galaxy.web import url_for
import tool_shed.util.shed_util_common as suc
from tool_shed.util import tool_util
import xml.etree.ElementTree
from tool_shed.util import xml_util
from galaxy import eggs
eggs.require( 'mercurial' )
@@ -60,17 +60,6 @@ def check_file_content_for_html_and_images( file_path ):
message = 'The file "%s" contains image content.\n' % str( file_path )
return message
def create_and_write_tmp_file( root ):
tmp_str = '%s\n' % xml.etree.ElementTree.tostring( root, encoding='utf-8' )
fh = tempfile.NamedTemporaryFile( 'wb' )
tmp_filename = fh.name
fh.close()
fh = open( tmp_filename, 'wb' )
fh.write( '<?xml version="1.0"?>\n' )
fh.write( tmp_str )
fh.close()
return tmp_filename
def get_upload_point( repository, **kwd ):
upload_point = kwd.get( 'upload_point', None )
if upload_point is not None:
@@ -198,7 +187,7 @@ def handle_repository_dependencies_definition( trans, repository_dependencies_co
altered = False
try:
# Make sure we're looking at a valid repository_dependencies.xml file.
tree = suc.parse_xml( repository_dependencies_config )
tree = xml_util.parse_xml( repository_dependencies_config )
root = tree.getroot()
except Exception, e:
error_message = "Error parsing %s in handle_repository_dependencies_definition: " % str( repository_dependencies_config )
@@ -245,7 +234,7 @@ def handle_tool_dependencies_definition( trans, tool_dependencies_config ):
altered = False
try:
# Make sure we're looking at a valid tool_dependencies.xml file.
tree = suc.parse_xml( tool_dependencies_config )
tree = xml_util.parse_xml( tool_dependencies_config )
root = tree.getroot()
except Exception, e:
error_message = "Error parsing %s in handle_tool_dependencies_definition: " % str( tool_dependencies_config )
+4 -4
View File
@@ -1,8 +1,8 @@
import os
import urllib2
from galaxy import util
from galaxy.util.odict import odict
from tool_shed.util import encoding_util
from tool_shed.util import xml_util
REPOSITORY_OWNER = 'devteam'
@@ -11,7 +11,7 @@ def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_scri
tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) )
# Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
migrated_tool_configs_dict = odict()
tree = util.parse_xml( tools_xml_file_path )
tree = xml_util.parse_xml( tools_xml_file_path )
root = tree.getroot()
tool_shed = root.get( 'name' )
tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed )
@@ -48,7 +48,7 @@ def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_scri
# Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names.
missing_tool_configs_dict = odict()
for tool_panel_config in tool_panel_configs:
tree = util.parse_xml( tool_panel_config )
tree = xml_util.parse_xml( tool_panel_config )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
@@ -78,7 +78,7 @@ def get_non_shed_tool_panel_configs( app ):
for config_filename in app.config.tool_configs:
# Any config file that includes a tool_path attribute in the root tag set like the following is shed-related.
# <toolbox tool_path="../shed_tools">
tree = util.parse_xml( config_filename )
tree = xml_util.parse_xml( config_filename )
root = tree.getroot()
tool_path = root.get( 'tool_path', None )
if tool_path is None:
+7 -7
View File
@@ -1,6 +1,6 @@
import logging
import os
from galaxy import util
from tool_shed.util import xml_util
import tool_shed.util.shed_util_common as suc
log = logging.getLogger( __name__ )
@@ -10,7 +10,7 @@ def data_manager_config_elems_to_xml_file( app, config_elems, config_filename ):
fh = open( config_filename, 'wb' )
fh.write( '<?xml version="1.0"?>\n<data_managers>\n' )#% ( shed_tool_conf_filename ))
for elem in config_elems:
fh.write( util.xml_to_string( elem, pretty=True ) )
fh.write( xml_util.xml_to_string( elem ) )
fh.write( '</data_managers>\n' )
fh.close()
@@ -21,7 +21,7 @@ def install_data_managers( app, shed_data_manager_conf_filename, metadata_dict,
for tool_tup in repository_tools_tups:
repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] )
# Load existing data managers.
config_elems = [ elem for elem in util.parse_xml( shed_data_manager_conf_filename ).getroot() ]
config_elems = [ elem for elem in xml_util.parse_xml( shed_data_manager_conf_filename ).getroot() ]
repo_data_manager_conf_filename = metadata_dict['data_manager'].get( 'config_filename', None )
if repo_data_manager_conf_filename is None:
log.debug( "No data_manager_conf.xml file has been defined." )
@@ -29,13 +29,13 @@ def install_data_managers( app, shed_data_manager_conf_filename, metadata_dict,
data_manager_config_has_changes = False
relative_repo_data_manager_dir = os.path.join( shed_config_dict.get( 'tool_path', '' ), relative_install_dir )
repo_data_manager_conf_filename = os.path.join( relative_repo_data_manager_dir, repo_data_manager_conf_filename )
tree = util.parse_xml( repo_data_manager_conf_filename )
tree = xml_util.parse_xml( repo_data_manager_conf_filename )
root = tree.getroot()
for elem in root:
if elem.tag == 'data_manager':
data_manager_id = elem.get( 'id', None )
if data_manager_id is None:
log.error( "A data manager was defined that does not have an id and will not be installed:\n%s" % ( util.xml_to_string( elem ) ) )
log.error( "A data manager was defined that does not have an id and will not be installed:\n%s" % ( xml_util.xml_to_string( elem ) ) )
continue
data_manager_dict = metadata_dict['data_manager'].get( 'data_managers', {} ).get( data_manager_id, None )
if data_manager_dict is None:
@@ -77,7 +77,7 @@ def install_data_managers( app, shed_data_manager_conf_filename, metadata_dict,
if data_manager:
rval.append( data_manager )
else:
log.warning( "Encountered unexpected element '%s':\n%s" % ( elem.tag, util.xml_to_string( elem ) ) )
log.warning( "Encountered unexpected element '%s':\n%s" % ( elem.tag, xml_util.xml_to_string( elem ) ) )
config_elems.append( elem )
data_manager_config_has_changes = True
# Persist the altered shed_data_manager_config file.
@@ -89,7 +89,7 @@ def remove_from_data_manager( app, repository ):
metadata_dict = repository.metadata
if metadata_dict and 'data_manager' in metadata_dict:
shed_data_manager_conf_filename = app.config.shed_data_manager_config_file
tree = util.parse_xml( shed_data_manager_conf_filename )
tree = xml_util.parse_xml( shed_data_manager_conf_filename )
root = tree.getroot()
assert root.tag == 'data_managers', 'The file provided (%s) for removing data managers from is not a valid data manager xml file.' % ( shed_data_manager_conf_filename )
guids = [ data_manager_dict.get( 'guid' ) for data_manager_dict in metadata_dict.get( 'data_manager', {} ).get( 'data_managers', {} ).itervalues() if 'guid' in data_manager_dict ]
+4 -4
View File
@@ -2,7 +2,7 @@ import logging
import os
import tempfile
from galaxy import eggs
from galaxy import util
from tool_shed.util import xml_util
import tool_shed.util.shed_util_common as suc
import pkg_resources
@@ -24,7 +24,7 @@ def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_
has been initialized, the registry's contents cannot be overridden by conflicting data types.
"""
try:
tree = util.parse_xml( datatypes_config )
tree = xml_util.parse_xml( datatypes_config )
except Exception, e:
log.debug( "Error parsing %s, exception: %s" % ( datatypes_config, str( e ) ) )
return None, None
@@ -83,9 +83,9 @@ def alter_config_and_load_prorietary_datatypes( app, datatypes_config, relative_
fd, proprietary_datatypes_config = tempfile.mkstemp()
os.write( fd, '<?xml version="1.0"?>\n' )
os.write( fd, '<datatypes>\n' )
os.write( fd, '%s' % util.xml_to_string( registration ) )
os.write( fd, '%s' % xml_util.xml_to_string( registration ) )
if sniffers:
os.write( fd, '%s' % util.xml_to_string( sniffers ) )
os.write( fd, '%s' % xml_util.xml_to_string( sniffers ) )
os.write( fd, '</datatypes>\n' )
os.close( fd )
os.chmod( proprietary_datatypes_config, 0644 )
+4 -3
View File
@@ -17,6 +17,7 @@ from tool_shed.util import common_install_util
from tool_shed.util import readme_util
from tool_shed.util import tool_dependency_util
from tool_shed.util import tool_util
from tool_shed.util import xml_util
import pkg_resources
@@ -363,7 +364,7 @@ def generate_data_manager_metadata( app, repository, repo_dir, data_manager_conf
'error_messages': [] }
metadata_dict[ 'data_manager' ] = data_manager_metadata
try:
tree = util.parse_xml( data_manager_config_filename )
tree = xml_util.parse_xml( data_manager_config_filename )
except Exception, e:
# We are not able to load any data managers.
error_message = 'There was an error parsing your Data Manager config file "%s": %s' % ( data_manager_config_filename, e )
@@ -638,7 +639,7 @@ def generate_metadata_for_changeset_revision( app, repository, changeset_revisio
or checkers.check_bz2( full_path )[ 0 ] or checkers.check_zip( full_path ) ):
try:
# Make sure we're looking at a tool config and not a display application config or something else.
element_tree = util.parse_xml( full_path )
element_tree = xml_util.parse_xml( full_path )
element_tree_root = element_tree.getroot()
is_tool = element_tree_root.tag == 'tool'
except Exception, e:
@@ -751,7 +752,7 @@ def generate_repository_dependency_metadata( app, repository_dependencies_config
error_message = ''
try:
# Make sure we're looking at a valid repository_dependencies.xml file.
tree = util.parse_xml( repository_dependencies_config )
tree = xml_util.parse_xml( repository_dependencies_config )
root = tree.getroot()
xml_is_valid = root.tag == 'repositories'
except Exception, e:
+4 -22
View File
@@ -14,7 +14,7 @@ from galaxy.datatypes import checkers
from galaxy.model.orm import and_
import sqlalchemy.orm.exc
from tool_shed.util import common_util
from xml.etree import ElementTree as XmlET
from tool_shed.util import xml_util
from galaxy import eggs
import pkg_resources
@@ -37,17 +37,6 @@ MAX_CONTENT_SIZE = 1048576
MAX_DISPLAY_SIZE = 32768
VALID_CHARS = set( string.letters + string.digits + "'\"-=_.()/+*^,:?!#[]%\\$@;{}&<>" )
class CommentedTreeBuilder ( XmlET.XMLTreeBuilder ):
def __init__ ( self, html=0, target=None ):
XmlET.XMLTreeBuilder.__init__( self, html, target )
self._parser.CommentHandler = self.handle_comment
def handle_comment ( self, data ):
self._target.start( XmlET.Comment, {} )
self._target.data( data )
self._target.end( XmlET.Comment )
new_repo_email_alert_template = """
Sharable link: ${sharable_link}
Repository name: ${repository_name}
@@ -188,7 +177,7 @@ def config_elems_to_xml_file( app, config_elems, config_filename, tool_path ):
os.write( fd, '<?xml version="1.0"?>\n' )
os.write( fd, '<toolbox tool_path="%s">\n' % str( tool_path ) )
for elem in config_elems:
os.write( fd, '%s' % util.xml_to_string( elem, pretty=True ) )
os.write( fd, '%s' % xml_util.xml_to_string( elem ) )
os.write( fd, '</toolbox>\n' )
os.close( fd )
shutil.move( filename, os.path.abspath( config_filename ) )
@@ -346,7 +335,7 @@ def generate_tool_panel_dict_from_shed_tool_conf_entries( app, repository ):
file_name = strip_path( tool_config )
guids_and_configs[ guid ] = file_name
# Parse the shed_tool_conf file in which all of this repository's tools are defined and generate the tool_panel_dict.
tree = util.parse_xml( shed_tool_conf )
tree = xml_util.parse_xml( shed_tool_conf )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
@@ -1084,13 +1073,6 @@ def parse_repository_dependency_tuple( repository_dependency_tuple, contains_err
prior_installation_required = util.asbool( str( prior_installation_required ) )
return tool_shed, name, owner, changeset_revision, prior_installation_required
def parse_xml( fname ):
"""Returns a parsed xml tree with comments in tact."""
fobj = open( fname, 'r' )
tree = XmlET.parse( fobj, parser=CommentedTreeBuilder() )
fobj.close()
return tree
def pretty_print( dict=None ):
if dict:
return json.to_json_string( dict, sort_keys=True, indent=4 * ' ' )
@@ -1271,7 +1253,7 @@ def update_in_shed_tool_config( app, repository ):
for tool_config_filename, guid, tool in repository_tools_tups:
guid_to_tool_elem_dict[ guid ] = generate_tool_elem( tool_shed, repository.name, repository.changeset_revision, repository.owner or '', tool_config_filename, tool, None )
config_elems = []
tree = util.parse_xml( shed_tool_conf )
tree = xml_util.parse_xml( shed_tool_conf )
root = tree.getroot()
for elem in root:
if elem.tag == 'section':
+4 -3
View File
@@ -12,6 +12,7 @@ from galaxy.tools import parameters
from galaxy.tools.parameters import dynamic_options
from galaxy.tools.search import ToolBoxSearch
from galaxy.web.form_builder import SelectField
from tool_shed.util import xml_util
import tool_shed.util.shed_util_common as suc
import pkg_resources
@@ -36,7 +37,7 @@ def add_to_shed_tool_config( app, shed_tool_conf_dict, elem_list ):
shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
tool_path = shed_tool_conf_dict[ 'tool_path' ]
config_elems = []
tree = util.parse_xml( shed_tool_conf )
tree = xml_util.parse_xml( shed_tool_conf )
root = tree.getroot()
for elem in root:
config_elems.append( elem )
@@ -803,7 +804,7 @@ def load_tool_from_tmp_config( trans, repo, repository_id, ctx, ctx_file, work_d
message = ''
tmp_tool_config = suc.get_named_tmpfile_from_ctx( ctx, ctx_file, work_dir )
if tmp_tool_config:
element_tree = util.parse_xml( tmp_tool_config )
element_tree = xml_util.parse_xml( tmp_tool_config )
element_tree_root = element_tree.getroot()
# Look for code files required by the tool config.
tmp_code_files = []
@@ -845,7 +846,7 @@ def remove_from_shed_tool_config( trans, shed_tool_conf_dict, guids_to_remove ):
shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
tool_path = shed_tool_conf_dict[ 'tool_path' ]
config_elems = []
tree = util.parse_xml( shed_tool_conf )
tree = xml_util.parse_xml( shed_tool_conf )
root = tree.getroot()
for elem in root:
config_elems.append( elem )
+43
View File
@@ -0,0 +1,43 @@
import logging
import os
import tempfile
from xml.etree import ElementTree as XmlET
import xml.etree.ElementTree
log = logging.getLogger( __name__ )
class CommentedTreeBuilder ( XmlET.XMLTreeBuilder ):
def __init__ ( self, html=0, target=None ):
XmlET.XMLTreeBuilder.__init__( self, html, target )
self._parser.CommentHandler = self.handle_comment
def handle_comment ( self, data ):
self._target.start( XmlET.Comment, {} )
self._target.data( data )
self._target.end( XmlET.Comment )
def create_and_write_tmp_file( elem ):
tmp_str = xml_to_string( elem )
fh = tempfile.NamedTemporaryFile( 'wb' )
tmp_filename = fh.name
fh.close()
fh = open( tmp_filename, 'wb' )
fh.write( '<?xml version="1.0"?>\n' )
fh.write( tmp_str )
fh.close()
return tmp_filename
def parse_xml( file_name ):
"""Returns a parsed xml tree with comments intact."""
try:
fobj = open( file_name, 'r' )
tree = XmlET.parse( fobj, parser=CommentedTreeBuilder() )
fobj.close()
except Exception, e:
log.exception( "Exception attempting to parse %s: %s" % ( str( file_name ), str( e ) ) )
return None
return tree
def xml_to_string( elem, encoding='utf-8' ):
return '%s\n' % xml.etree.ElementTree.tostring( elem, encoding=encoding )