opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-24 02:04:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Convert everything from elementtree to stdlib xml module.

Browse Source
This commit is contained in:
Dannon Baker
2015-01-29 16:10:56 -05:00
parent ec3eec5609
commit dba89dff83
24 changed files with 68 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cron/parse_builds.py
+2 -7
View File
@@ -8,12 +8,7 @@ build description
import sys
import urllib
if sys.version_info[:2] >= ( 2, 5 ):
import xml.etree.ElementTree as ElementTree
else:
from galaxy import eggs
import pkg_resources; pkg_resources.require( "elementtree" )
from elementtree import ElementTree
import xml.etree.ElementTree as ElementTree
URL = "http://genome.cse.ucsc.edu/cgi-bin/das/dsn"
@@ -38,7 +33,7 @@ def getbuilds(url):
for dsn in tree:
build = dsn.find("SOURCE").attrib['id']
description = dsn.find("DESCRIPTION").text.replace(" - Genome at UCSC","").replace(" Genome at UCSC","")
fields = description.split(" ")
temp = fields[0]
for i in range(len(fields)-1):
cron/parse_builds_3_sites.py
+2 -7
View File
@@ -5,12 +5,7 @@ Connects to sites and determines which builds are available at each.
import sys
import urllib
if sys.version_info[:2] >= ( 2, 5 ):
import xml.etree.ElementTree as ElementTree
else:
from galaxy import eggs
import pkg_resources; pkg_resources.require( "elementtree" )
from elementtree import ElementTree
import xml.etree.ElementTree as ElementTree
sites = ['http://genome.ucsc.edu/cgi-bin/',
'http://archaea.ucsc.edu/cgi-bin/',
@@ -38,7 +33,7 @@ def main():
print "#Invalid xml passed back from " + site
continue
print "#Harvested from",site
for dsn in tree:
build = dsn.find("SOURCE").attrib['id']
builds.append(build)
eggs.ini
-1
View File
@@ -38,7 +38,6 @@ boto = 2.27.0
decorator = 3.1.2
docutils = 0.7
drmaa = 0.7.6
elementtree = 1.2.6_20050316
Fabric = 1.7.0
importlib = 1.0.3
kombu = 3.0.13
lib/galaxy/datatypes/dataproviders/hierarchy.py
+7 -7
View File
@@ -3,7 +3,7 @@ Dataproviders that iterate over lines from their sources.
"""
import line
import xml.etree.ElementTree as elementtree
from xml.etree.ElementTree import Element, iterparse
_TODO = """
"""
@@ -30,7 +30,7 @@ class XMLDataProvider( HierarchalDataProvider ):
"""
Data provider that converts selected XML elements to dictionaries.
"""
# using elementtree's iterparse method to keep mem down
# using xml.etree's iterparse method to keep mem down
#TODO: this, however (AFAIK), prevents the use of xpath
settings = {
'selector' : 'str', #urlencoded
@@ -54,7 +54,7 @@ class XMLDataProvider( HierarchalDataProvider ):
"""
Returns true if the ``element`` matches the ``selector``.
:param element: an XML ``ElementTree.Element``
:param element: an XML ``Element``
:param selector: some partial string in the desired tags to return
Change point for more sophisticated selectors.
@@ -64,13 +64,13 @@ class XMLDataProvider( HierarchalDataProvider ):
#TODO: fails with '#' - browser thinks it's an anchor - use urlencode
#TODO: need removal/replacement of etree namespacing here - then move to string match
return bool( ( selector == None )
or ( isinstance( element, elementtree.Element ) and selector in element.tag ) )
or ( isinstance( element, Element ) and selector in element.tag ) )
def element_as_dict( self, element ):
"""
Converts an XML element (its text, tag, and attributes) to dictionary form.
:param element: an XML ``ElementTree.Element``
:param element: an XML ``Element``
"""
#TODO: Key collision is unlikely here, but still should be better handled
return {
@@ -84,7 +84,7 @@ class XMLDataProvider( HierarchalDataProvider ):
"""
Yield all children of element (and their children - recursively)
in dictionary form.
:param element: an XML ``ElementTree.Element``
:param element: an XML ``Element``
:param max_depth: the number of generations of descendents to return
"""
if not isinstance( max_depth, int ) or max_depth >= 1:
@@ -99,7 +99,7 @@ class XMLDataProvider( HierarchalDataProvider ):
yield child_data
def __iter__( self ):
context = elementtree.iterparse( self.source, events=self.ITERPARSE_ALL_EVENTS )
context = iterparse( self.source, events=self.ITERPARSE_ALL_EVENTS )
context = iter( context )
selected_element = None
lib/galaxy/sample_tracking/external_service_types.py
-1
View File
@@ -4,7 +4,6 @@ from galaxy.util.bunch import Bunch
from galaxy import util, jobs, model
from galaxy.forms.forms import form_factory
from galaxy.external_services.service import ExternalServiceActionsGroup
from elementtree.ElementTree import XML
from galaxy.sample_tracking.data_transfer import data_transfer_factories
log = logging.getLogger( __name__ )
lib/galaxy/tools/__init__.py
+17 -18
View File
@@ -18,12 +18,11 @@ from galaxy import eggs, util
eggs.require( "MarkupSafe" ) # MarkupSafe must load before mako
eggs.require( "Mako" )
eggs.require( "elementtree" )
eggs.require( "Paste" )
eggs.require( "SQLAlchemy >= 0.4" )
from cgi import FieldStorage
from elementtree import ElementTree
from xml.etree import ElementTree
from mako.template import Template
from paste import httpexceptions
@@ -2280,7 +2279,7 @@ class Tool( object, Dictifiable ):
trans.response.status = 500
log.error('Failed to get job information.')
return { 'error': 'Failed to get job information.' }
# load job parameters into incoming
tool_message = ''
if job:
@@ -2295,7 +2294,7 @@ class Tool( object, Dictifiable ):
# create parameter object
params = galaxy.util.Params( kwd, sanitize = False )
# convert value to jsonifiable value
def jsonify(v):
# check if value is numeric
@@ -2344,16 +2343,16 @@ class Tool( object, Dictifiable ):
# update and return
dict[key] = value
# check the current state of a value and update it if necessary
def check_state(trans, input, default_value, context):
value = default_value
error = 'State validation failed.'
# skip dynamic fields if deactivated
if not is_dynamic and input.is_dynamic:
return [value, None]
# validate value content
try:
# resolves the inconsistent definition of boolean parameters (see base.py) without modifying shared code
@@ -2365,7 +2364,7 @@ class Tool( object, Dictifiable ):
log.error('Checking parameter failed. %s', str(err))
pass
return [value, error]
# populates state with incoming url parameters
def populate_state(trans, inputs, state, errors, incoming, prefix="", context=None ):
context = ExpressionContext(state, context)
@@ -2410,7 +2409,7 @@ class Tool( object, Dictifiable ):
if error:
errors[key] = error
state[input.name] = value
# builds tool model including all attributes
def iterate(group_inputs, inputs, state_inputs, other_values=None):
other_values = ExpressionContext( state_inputs, other_values )
@@ -2442,13 +2441,13 @@ class Tool( object, Dictifiable ):
tool_dict = input.to_dict(trans, other_values=other_values)
except Exception:
pass
# identify name
input_name = tool_dict.get('name')
if input_name:
# backup default value
tool_dict['default_value'] = input.get_initial_value(trans, other_values)
# update input value from tool state
if input_name in state_inputs:
tool_dict['value'] = state_inputs[input_name]
@@ -2481,7 +2480,7 @@ class Tool( object, Dictifiable ):
# do param translation here, used by datasource tools
if self.input_translator:
self.input_translator.translate( params )
# initialize and populate tool state
state_inputs = {}
state_errors = {}
@@ -2490,7 +2489,7 @@ class Tool( object, Dictifiable ):
# create basic tool model
tool_model = self.to_dict(trans)
tool_model['inputs'] = {}
# build tool model and tool state
iterate(tool_model['inputs'], self.inputs, state_inputs, '')
@@ -2504,18 +2503,18 @@ class Tool( object, Dictifiable ):
tool_help = tool_help.render( static_path=url_for( '/static' ), host_url=url_for('/', qualified=True) )
if type( tool_help ) is not unicode:
tool_help = unicode( tool_help, 'utf-8')
# check if citations exist
tool_citations = False
if self.citations:
tool_citations = True
# get tool versions
tool_versions = []
tools = self.app.toolbox.get_loaded_tools_by_lineage(self.id)
for t in tools:
tool_versions.append(t.version)
## add information with underlying requirements and their versions
tool_requirements = []
if self.requirements:
@@ -2577,13 +2576,13 @@ class Tool( object, Dictifiable ):
except Exception, error:
trans.response.status = 500
return { 'error': str (error) }
# can't rerun upload, external data sources, et cetera. workflow compatible will proxy this for now
#if not self.is_workflow_compatible:
# trans.response.status = 500
# return { 'error': 'The \'%s\' tool does currently not support re-running.' % self.name }
return message
def get_default_history_by_trans( self, trans, create=False ):
return trans.get_history( create=create )
lib/galaxy/tools/parameters/basic.py
+1 -1
View File
@@ -8,7 +8,7 @@ import sys
import os
import os.path
import urllib
from elementtree.ElementTree import XML, Element
from xml.etree.ElementTree import XML
from galaxy import config, datatypes, util
from galaxy.web import form_builder
from galaxy.util.bunch import Bunch
lib/galaxy/tools/parameters/input_translation.py
+1 -1
View File
@@ -13,7 +13,7 @@ class ToolInputTranslator( object ):
This is used for data source tools
>>> from galaxy.util import Params
>>> from elementtree.ElementTree import XML
>>> from xml.etree.ElementTree import XML
>>> translator = ToolInputTranslator.from_element( XML(
... '''
... <request_param_translation>
lib/galaxy/tools/parameters/sanitize.py
+1 -1
View File
@@ -12,7 +12,7 @@ class ToolParameterSanitizer( object ):
"""
Handles tool parameter specific sanitizing.
>>> from elementtree.ElementTree import XML
>>> from xml.etree.ElementTree import XML
>>> sanitizer = ToolParameterSanitizer.from_element( XML(
... '''
... <sanitizer invalid_char="">
lib/galaxy/tools/parameters/validation.py
+1 -1
View File
@@ -3,7 +3,7 @@ Classes related to parameter validation.
"""
import os, re, logging
from elementtree.ElementTree import XML
from xml.etree.ElementTree import XML
from galaxy import model
log = logging.getLogger( __name__ )
lib/galaxy/util/__init__.py
+1 -2
View File
@@ -40,8 +40,7 @@ eggs.require( 'docutils' )
import docutils.core
import docutils.writers.html4css1
eggs.require( 'elementtree' )
from elementtree import ElementTree, ElementInclude
from xml.etree import ElementTree, ElementInclude
eggs.require( "wchartype" )
import wchartype
lib/galaxy/webapps/galaxy/api/forms.py
+1 -1
View File
@@ -5,7 +5,7 @@ import logging
from galaxy.web.base.controller import BaseAPIController, url_for
from galaxy import web
from galaxy.forms.forms import form_factory
from elementtree.ElementTree import XML
from xml.etree.ElementTree import XML
log = logging.getLogger( __name__ )
lib/galaxy/webapps/galaxy/api/request_types.py
+1 -1
View File
@@ -5,7 +5,7 @@ import logging
from galaxy.web.base.controller import BaseAPIController, url_for
from galaxy import web
from galaxy.sample_tracking.request_types import request_type_factory
from elementtree.ElementTree import XML
from xml.etree.ElementTree import XML
log = logging.getLogger( __name__ )
lib/galaxy/webapps/galaxy/controllers/forms.py
+2 -2
View File
@@ -5,7 +5,7 @@ from galaxy import model, util
import logging, os, sys
from galaxy.web.form_builder import *
from galaxy.tools.parameters.basic import parameter_types
from elementtree.ElementTree import XML, Element
from xml.etree.ElementTree import XML, Element
from galaxy.util.odict import odict
import copy
from galaxy.web.framework.helpers import time_ago, iff, grids
@@ -65,7 +65,7 @@ class FormsGrid( grids.Grid ):
global_actions = [
grids.GridAction( "Create new form", dict( controller='forms', action='create_form_definition' ) )
]
def build_initial_query( self, trans, **kwargs ):
return trans.sa_session.query( self.model_class ).join (model.FormDefinition, self.model_class.latest_form_id == model.FormDefinition.id)
lib/galaxy/workflow/modules.py
+1 -2
View File
@@ -6,9 +6,8 @@ import logging
import re
from galaxy import eggs
eggs.require( "elementtree" )
from elementtree.ElementTree import Element
from xml.etree.ElementTree import Element
import galaxy.tools
from galaxy import exceptions
scripts/loc_files/create_all_fasta_loc.py
+2 -2
View File
@@ -1,5 +1,5 @@
import optparse, os, sys
import elementtree.ElementTree as etree
from xml.etree.ElementTree import parse
"""
Generates a loc file containing names of all the fasta files that match the
@@ -212,7 +212,7 @@ def __main__():
# get column names
col_values = []
loc_path = None
tree = etree.parse( options.data_table_xml )
tree = parse( options.data_table_xml )
tables = tree.getroot()
for table in tables.getiterator():
name = table.attrib.get( 'name' )
scripts/microbes/ncbi_to_ucsc.py
+16 -16
View File
@@ -7,7 +7,7 @@ Uses UCSC Archaea DSN.
import sys, os
import urllib
from elementtree import ElementTree
from xml.etree import ElementTree
from BeautifulSoup import BeautifulSoup
from shutil import move
@@ -17,7 +17,7 @@ def __main__():
base_dir = sys.argv[1]
except:
print "using default base_dir:", base_dir
organisms = {}
for result in os.walk(base_dir):
this_base_dir,sub_dirs,files = result
@@ -76,7 +76,7 @@ def __main__():
org_page.pop(0)
if org_page[-1]=="":
org_page.pop(-1)
for row in org_page:
chr = row.split("</a>")[0].split(">")[-1]
refseq = row.split("</a>")[-2].split(">")[-1]
@@ -87,20 +87,20 @@ def __main__():
builds[org]={'chrs':{},'build':build}
builds[org]['chrs'][refseq]=chr
#print build,org,chr,refseq
print
ext_to_edit = ['bed', 'info', ]
for org in builds:
print org,"changed to",builds[org]['build']
#org info file
info_file_old = os.path.join(base_dir+org,org+".info")
info_file_new = os.path.join(base_dir+org,builds[org]['build']+".info")
old_dir = base_dir+org
new_dir = base_dir+builds[org]['build']
#open and edit org info file
info_file_contents = open(info_file_old).read()
info_file_contents = info_file_contents+"build="+builds[org]['build']+"\n"
@@ -114,31 +114,31 @@ def __main__():
old_name = os.path.join(this_base_dir,file)
new_name = os.path.join(this_base_dir,builds[org]['chrs'][chrom]+file[len(chrom):])
move(old_name,new_name)
#edit contents of file, skiping those in list
if file.split(".")[-1] not in ext_to_edit: continue
file_contents = open(new_name).read()
file_contents = file_contents.replace(chrom,builds[org]['chrs'][chrom])
#special case fixes...
if file[-5:] == ".info":
file_contents = file_contents.replace("organism="+org,"organism="+builds[org]['build'])
file_contents = file_contents.replace("refseq="+builds[org]['chrs'][chrom],"refseq="+chrom)
#write out new file
file_out = open(new_name,'w')
file_out.write(file_contents)
file_out.close()
#write out org info file and remove old file
org_info_out = open(info_file_new,'w')
org_info_out.write(info_file_contents)
org_info_out.close()
os.unlink(info_file_old)
#change org directory name
move(old_dir,new_dir)
test/base/asserts/xml.py
+5 -6
View File
@@ -1,10 +1,10 @@
import elementtree.ElementTree
import xml.etree
import re
# Helper functions used to work with XML output.
def to_xml(output):
return elementtree.ElementTree.fromstring(output)
return xml.etree.fromstring(output)
def xml_find_text(output, path):
@@ -31,8 +31,7 @@ def assert_is_valid_xml(output):
def assert_has_element_with_path(output, path):
""" Asserts the specified output has at least one XML element with a
path matching the specified path argument. Valid paths are the
simplified subsets of XPath implemented by elementtree (currently
Galaxy makes use of elementtree 1.2). See
simplified subsets of XPath implemented by xml.etree;
http://effbot.org/zone/element-xpath.htm for more information."""
if xml_find(output, path) is None:
errmsg = "Expected to find XML element matching expression %s, not such match was found." % path
@@ -74,13 +73,13 @@ def assert_attribute_matches(output, path, attribute, expression):
errmsg = "Expected attribute '%s' on element with path '%s' to match '%s', instead attribute value was '%s'." % (attribute, path, expression, attribute_value)
raise AssertionError(errmsg)
def assert_attribute_is(output, path, attribute, text):
""" Asserts the specified attribute of the first element matching
the specified path matches exactly the specified text."""
assert_attribute_matches(output, path, attribute, re.escape(text))
def assert_element_text(output, path, verify_assertions_function, children):
""" Recursively checks the specified assertions against the text of
the first element matching the specified path."""
test/base/twilltestcase.py
+1 -2
View File
@@ -22,10 +22,9 @@ from galaxy.web.framework.helpers import iff, escape
from urlparse import urlparse
from galaxy import eggs
eggs.require( "elementtree" )
eggs.require( 'twill' )
from elementtree import ElementTree
from xml.etree import ElementTree
import twill
import twill.commands as tc
test/unit/tools/test_actions.py
+1 -1
View File
@@ -6,7 +6,7 @@ from galaxy.tools import ToolOutput
from galaxy.tools.actions import DefaultToolAction
from galaxy.tools.actions import on_text_for_names
from galaxy.tools.actions import determine_output_format
from elementtree.ElementTree import XML
from xml.etree.ElementTree import XML
import tools_support
test/unit/tools/test_dataset_matcher.py
+1 -2
View File
@@ -1,12 +1,11 @@
from unittest import TestCase
from xml.etree.ElementTree import XML
from galaxy import model
from galaxy.util import bunch
from galaxy.tools.parameters import basic
from galaxy.tools.parameters import dataset_matcher
from elementtree.ElementTree import XML
import tools_support
from .test_data_parameters import MockHistoryDatasetAssociation
test/unit/tools/test_evaluation.py
+2 -2
View File
@@ -1,6 +1,7 @@
import os
from xml.etree.ElementTree import XML
from unittest import TestCase
from galaxy.model import Job
from galaxy.model import History
from galaxy.model import Dataset
@@ -21,7 +22,6 @@ from galaxy.tools.parameters.grouping import ConditionalWhen
from galaxy.tools.parameters.basic import IntegerToolParameter
from galaxy.tools.parameters.basic import SelectToolParameter
from galaxy.tools.parameters.basic import DataToolParameter
from elementtree.ElementTree import XML # Import after model, to ensure elementtree
# Test fixtures for Galaxy infrastructure.
test/unit/tools/test_parameter_parsing.py
+1 -1
View File
@@ -2,7 +2,7 @@ from unittest import TestCase
from galaxy.tools.parameters import basic
from galaxy.util import bunch
from elementtree.ElementTree import XML
from xml.etree.ElementTree import XML
import tools_support
test/unit/tools/test_wrappers.py
+1 -1
View File
@@ -10,7 +10,7 @@ from galaxy.tools.wrappers import InputValueWrapper
from galaxy.tools.wrappers import DatasetFilenameWrapper
from galaxy.jobs.datasets import DatasetPath
from galaxy.util.bunch import Bunch
from elementtree.ElementTree import XML
from xml.etree.ElementTree import XML
from galaxy.datatypes.metadata import MetadataSpecCollection