diff --git a/datatype_converters_conf.xml.sample b/datatype_converters_conf.xml.sample
deleted file mode 100644
index 2ef823b5268..00000000000
--- a/datatype_converters_conf.xml.sample
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
diff --git a/datatypes_conf.xml.sample b/datatypes_conf.xml.sample
new file mode 100644
index 00000000000..027d3365c6d
--- /dev/null
+++ b/datatypes_conf.xml.sample
@@ -0,0 +1,163 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py
index 1a969cae32d..0c4ea728395 100644
--- a/lib/galaxy/app.py
+++ b/lib/galaxy/app.py
@@ -13,9 +13,9 @@ class UniverseApplication( object ):
self.config = config.Configuration( **kwargs )
self.config.check()
config.configure_logging( self.config )
- #Set up datatypes registry
- self.datatypes_registry = galaxy.datatypes.registry.Registry(datatypes=self.config.datatypes, sniff_order=self.config.sniff_order)
- galaxy.model.set_datatypes_registry(self.datatypes_registry)
+ # Set up datatypes registry
+ self.datatypes_registry = galaxy.datatypes.registry.Registry( self.config.root, self.config.datatypes_config )
+ galaxy.model.set_datatypes_registry( self.datatypes_registry )
# Determine the database url
if self.config.database_connection:
db_url = self.config.database_connection
@@ -29,7 +29,7 @@ class UniverseApplication( object ):
# Initialize the tools
self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path, self )
#Load datatype converters
- self.datatypes_registry.load_datatype_converters(self.config.datatype_converters_config, self.config.datatype_converters_path, self.toolbox)
+ self.datatypes_registry.load_datatype_converters( self.toolbox )
# Start the job queue
job_dispatcher = jobs.DefaultJobDispatcher( self )
self.job_queue = jobs.JobQueue( self, job_dispatcher )
diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py
index 909a6cea43d..49989839285 100644
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -72,27 +72,16 @@ class Configuration( object ):
self.tool_runners = global_conf_parser.items("galaxy:tool_runners")
except ConfigParser.NoSectionError:
self.tool_runners = []
- #Store datatypes config
- try:
- self.datatypes = global_conf_parser.items("galaxy:datatypes")
- except ConfigParser.NoSectionError:
- self.datatypes = []
- #Store sniff order config
- try:
- self.sniff_order = global_conf_parser.items("galaxy:sniff_order")
- except ConfigParser.NoSectionError:
- self.sniff_order = []
- self.datatype_converters_config = kwargs.get( 'datatype_converters_config_file', "datatype_converters_conf.xml" )
- self.datatype_converters_path = kwargs.get( 'datatype_converters_path', os.path.join(self.root,"lib/galaxy/datatypes/converters") )
+ self.datatypes_config = kwargs.get( 'datatypes_config_file', 'datatypes_conf.xml' )
def get( self, key, default ):
return self.config_dict.get( key, default )
def check( self ):
# Check that required directories exist
- for path in self.root, self.file_path, self.tool_path, self.tool_data_path, self.template_path, self.job_working_directory, self.datatype_converters_path:
+ for path in self.root, self.file_path, self.tool_path, self.tool_data_path, self.template_path, self.job_working_directory:
if not os.path.isdir( path ):
raise ConfigurationError("Directory does not exist: %s" % path )
# Check that required files exist
- for path in self.tool_config, self.datatype_converters_config:
+ for path in self.tool_config, self.datatypes_config:
if not os.path.isfile(path):
raise ConfigurationError("File not found: %s" % path )
diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py
index 82559c74408..4efa3da5922 100644
--- a/lib/galaxy/datatypes/registry.py
+++ b/lib/galaxy/datatypes/registry.py
@@ -3,61 +3,92 @@ Provides mapping between extensions and datatypes, mime-types, etc.
"""
import os
import logging
-import data, tabular, interval, images, sequence, qualityscore
-import genetics # needed for rgenetics tools
+import data, tabular, interval, images, sequence, qualityscore, genetics
import galaxy.util
from galaxy.util.odict import odict
+class ConfigurationError( Exception ):
+ pass
+
class Registry( object ):
- def __init__( self, datatypes=[], sniff_order=[] ):
+ def __init__( self, root_dir=None, config=None ):
self.log = logging.getLogger(__name__)
self.datatypes_by_extension = {}
self.mimetypes_by_extension = {}
self.datatype_converters = odict()
- self.upload_file_formats = []
+ self.converters = []
self.sniff_order = []
- for ext, kind in datatypes:
- # Data types are defined in the config like this:
- # # = ,,
- try:
- fields = kind.split(",")
- kind = fields[0].strip()
- mime_type = None
- display_in_upload = False
- # See if we have a mime type or a display_in_upload
+ self.upload_file_formats = []
+ if root_dir and config:
+ # Parse datatypes_conf.xml
+ tree = galaxy.util.parse_xml( config )
+ root = tree.getroot()
+ # Load datatypes and converters from config
+ self.log.debug( 'Loading datatypes from %s' % config )
+ registration = root.find( 'registration' )
+ self.datatype_converters_path = os.path.join( root_dir, registration.get( 'converters_path', 'lib/galaxy/datatypes/converters' ) )
+ if not os.path.isdir( self.datatype_converters_path ):
+ raise ConfigurationError( "Directory does not exist: %s" % self.datatype_converters_path )
+ for elem in registration.findall( 'datatype' ):
try:
- ele = fields[1].strip()
- if ele:
- if ele == 'display_in_upload':
- display_in_upload = True
- else:
- mime_type = ele
- except:
- pass
- # See if we have a display_in_upload
- if not display_in_upload:
- try:
- ele = fields[2].strip()
- if ele == 'display_in_upload':
- display_in_upload = True
- except:
- pass
- if display_in_upload:
- self.upload_file_formats.append( ext )
- fields = kind.split(":")
- datatype_module = fields[0]
- datatype_class = fields[1]
- fields = datatype_module.split(".")
- module = __import__( fields.pop(0) )
- for mod in fields:
- module = getattr(module,mod)
- self.datatypes_by_extension[ext] = getattr(module, datatype_class)()
- if mime_type is None:
- # Use default mime type as per datatype spec
- mime_type = self.datatypes_by_extension[ext].get_mime()
- self.mimetypes_by_extension[ext] = mime_type
- except Exception, e:
- self.log.warning('error loading datatype "%s", problem: %s' % ( ext, str( e ) ) )
+ extension = elem.get( 'extension', None )
+ type = elem.get( 'type', None )
+ mimetype = elem.get( 'mimetype', None )
+ display_in_upload = elem.get( 'display_in_upload', False )
+ if extension and type:
+ fields = type.split( ':' )
+ datatype_module = fields[0]
+ datatype_class = fields[1]
+ fields = datatype_module.split( '.' )
+ module = __import__( fields.pop(0) )
+ for mod in fields:
+ module = getattr( module, mod )
+ self.datatypes_by_extension[extension] = getattr( module, datatype_class )()
+ if mimetype is None:
+ # Use default mime type as per datatype spec
+ mimetype = self.datatypes_by_extension[extension].get_mime()
+ self.mimetypes_by_extension[extension] = mimetype
+ if display_in_upload:
+ self.upload_file_formats.append( extension )
+ for converter in elem.findall( 'converter' ):
+ # Build the list of datatype converters which will later be loaded
+ # into the calling app's toolbox.
+ converter_config = converter.get( 'file', None )
+ target_datatype = converter.get( 'target_datatype', None )
+ if converter_config and target_datatype:
+ self.converters.append( ( converter_config, extension, target_datatype ) )
+ except Exception, e:
+ self.log.warning( 'Error loading datatype "%s", problem: %s' % ( extension, str( e ) ) )
+ # Load datatype sniffers from config
+ sniff_order = []
+ sniffers = root.find( 'sniffers' )
+ for elem in sniffers.findall( 'sniffer' ):
+ order = elem.get( 'order', None )
+ type = elem.get( 'type', None )
+ if order and type:
+ sniff_order.append( ( order, type ) )
+ sniff_order.sort()
+ for ele in sniff_order:
+ try:
+ type = ele[1]
+ fields = type.split( ":" )
+ datatype_module = fields[0]
+ datatype_class = fields[1]
+ fields = datatype_module.split( "." )
+ module = __import__( fields.pop(0) )
+ for mod in fields:
+ module = getattr( module, mod )
+ aclass = getattr( module, datatype_class )()
+ included = False
+ for atype in self.sniff_order:
+ if not issubclass( atype.__class__, aclass.__class__ ) and isinstance( atype, aclass.__class__ ):
+ included = True
+ break
+ if not included:
+ self.sniff_order.append( aclass )
+ self.log.debug( 'Loaded sniffer for datatype: %s' % type )
+ except Exception, exc:
+ self.log.warning( 'Error appending datatype %s to sniff_order, problem: %s' % ( type, str( exc ) ) )
#default values
if len(self.datatypes_by_extension) < 1:
self.datatypes_by_extension = {
@@ -104,33 +135,8 @@ class Registry( object ):
'txtseq.zip' : 'application/zip',
'wig' : 'text/plain'
}
- """
- The order in which we attempt to determine data types is critical
- because some formats are much more flexibly defined than others.
- """
- sniff_order.sort()
- for ele in sniff_order:
- try:
- ord = ele[0]
- kind = ele[1]
- fields = kind.split( ":" )
- datatype_module = fields[0]
- datatype_class = fields[1]
- fields = datatype_module.split( "." )
- module = __import__( fields.pop(0) )
- for mod in fields:
- module = getattr( module, mod )
- aclass = getattr( module, datatype_class )()
- included = False
- for atype in self.sniff_order:
- if not issubclass( atype.__class__, aclass.__class__ ) and isinstance( atype, aclass.__class__ ):
- included = True
- break
- if not included:
- self.sniff_order.append( aclass )
- except Exception, exc:
- self.log.warning( 'error appending datatype: %s to sniff_order, error: %s' % ( str( kind ), str( exc ) ) )
- #default values
+ # Default values - the order in which we attempt to determine data types is critical
+ # because some formats are much more flexibly defined than others.
if len(self.sniff_order) < 1:
self.sniff_order = [
sequence.Maf(),
@@ -146,7 +152,7 @@ class Registry( object ):
interval.Interval()
]
def append_to_sniff_order():
- """Just in case any supported data types are not included in the config's sniff_order section."""
+ # Just in case any supported data types are not included in the config's sniff_order section.
for ext in self.datatypes_by_extension:
datatype = self.datatypes_by_extension[ext]
included = False
@@ -157,7 +163,7 @@ class Registry( object ):
if not included:
self.sniff_order.append(datatype)
append_to_sniff_order()
-
+
def get_mimetype_by_extension(self, ext ):
"""Returns a mimetype based on an extension"""
try:
@@ -196,24 +202,20 @@ class Registry( object ):
setattr(newdata, key, value)
newdata.ext = ext
return newdata
-
- def load_datatype_converters(self, datatype_converters_config, datatype_converters_path, toolbox):
- """Loads datatype converters from a file, and adds to the toolbox"""
- self.datatype_converters = odict()
- tree = galaxy.util.parse_xml( datatype_converters_config )
- root = tree.getroot()
- self.log.debug( "Loading converters from %s" % (datatype_converters_config) )
- for elem in root.findall("converter"):
- path = elem.get("file")
- source_datatype = elem.get("source_datatype").split(",")
- target_datatype = elem.get("target_datatype")
- converter = toolbox.load_tool( os.path.join( datatype_converters_path, path ) )
- self.log.debug( "Loaded converter: %s", converter.id )
+
+ def load_datatype_converters( self, toolbox ):
+ """Adds datatype converters from self.converters to the calling app's toolbox"""
+ for elem in self.converters:
+ tool_config = elem[0]
+ source_datatype = elem[1]
+ target_datatype = elem[2]
+ converter = toolbox.load_tool( os.path.join( self.datatype_converters_path, tool_config ) )
toolbox.tools_by_id[converter.id] = converter
- for source_d in source_datatype:
- if source_d not in self.datatype_converters:
- self.datatype_converters[source_d] = odict()
- self.datatype_converters[source_d][target_datatype] = converter
+ if source_datatype not in self.datatype_converters:
+ self.datatype_converters[source_datatype] = odict()
+ self.datatype_converters[source_datatype][target_datatype] = converter
+ self.log.debug( "Loaded converter: %s", converter.id )
+
def get_converters_by_datatype(self, ext):
"""Returns available converters by source type"""
converters = odict()
diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample
index 39242d103e3..a5b262d436b 100644
--- a/universe_wsgi.ini.sample
+++ b/universe_wsgi.ini.sample
@@ -92,7 +92,7 @@ use_new_layout = true
# Comma separated list of UCSC / gbrowse browsers to use for viewing
ucsc_display_sites = main,test,archaea
-gbrowse_display_sites = wormbase,flybase,elegans
+gbrowse_display_sites = elegans,flybase
# Serving static files (needed if running standalone)
static_enabled = True
@@ -117,7 +117,7 @@ static_style_dir = %(here)s/static/june_2007_style/blue
# ---- Job Runners ----------------------------------------------------------
# Clustering Galaxy is not a straightforward process and requires a lot of
-# pre-configuration. Please see the Galaxy Wiki before attempting to set any
+# pre-configuration. See the ClusteringGalaxy Wiki before attempting to set any
# of these options. If running normally (without a cluster), do not change
# anything in this section.
@@ -129,10 +129,11 @@ static_style_dir = %(here)s/static/june_2007_style/blue
# default_cluster_job_runner: The URL for the default runner to use when a tool
# doesn't explicity define a runner below. For help on the cluster URL format,
-# see the Galaxy Wiki. Leave commented if not using a cluster job runner.
+# see the ClusteringGalaxy Wiki. Leave commented if not using a cluster job runner.
#default_cluster_job_runner = pbs:///
-# The PBS options are described in detail on the Galaxy Wiki
+# The PBS options are described in detail in the Galaxy Configuration section of
+# the ClusteringGalaxy Wiki
#pbs_application_server =
#pbs_stage_path =
#pbs_dataset_server =
@@ -162,151 +163,3 @@ ucsc_table_direct1 = local:///
ucsc_table_direct_archaea1 = local:///
ucsc_table_direct_test1 = local:///
upload1 = local:///
-
-# ---- Datatypes ------------------------------------------------------------
-
-[galaxy:datatypes]
-
-# = ,,
-ab1 = galaxy.datatypes.images:Ab1,application/octet-stream,display_in_upload
-axt = galaxy.datatypes.sequence:Axt,display_in_upload
-bed = galaxy.datatypes.interval:Bed,display_in_upload
-binseq.zip = galaxy.datatypes.images:Binseq,application/zip,display_in_upload
-customtrack = galaxy.datatypes.interval:CustomTrack
-data = galaxy.datatypes.data:Data,application/octet-stream
-fasta = galaxy.datatypes.sequence:Fasta,display_in_upload
-fastqsolexa = galaxy.datatypes.sequence:FastqSolexa,display_in_upload
-gff = galaxy.datatypes.interval:Gff,display_in_upload
-gff3 = galaxy.datatypes.interval:Gff3,display_in_upload
-gif = galaxy.datatypes.images:Image,image/gif
-gmaj.zip = galaxy.datatypes.images:Gmaj,application/zip
-html = galaxy.datatypes.images:Html,text/html
-interval = galaxy.datatypes.interval:Interval,display_in_upload
-jpg = galaxy.datatypes.images:Image,image/jpeg
-laj = galaxy.datatypes.images:Laj
-lav = galaxy.datatypes.sequence:Lav,display_in_upload
-maf = galaxy.datatypes.sequence:Maf,display_in_upload
-pdf = galaxy.datatypes.images:Image,application/pdf
-png = galaxy.datatypes.images:Image,image/png
-qual = galaxy.datatypes.qualityscore:QualityScore,display_in_upload
-scf = galaxy.datatypes.images:Scf,application/octet-stream,display_in_upload
-taxonomy = galaxy.datatypes.tabular:Taxonomy,display_in_upload
-tabular = galaxy.datatypes.tabular:Tabular,display_in_upload
-txt = galaxy.datatypes.data:Text,display_in_upload
-txtseq.zip = galaxy.datatypes.images:Txtseq,application/zip,display_in_upload
-wig = galaxy.datatypes.interval:Wiggle,display_in_upload
-#EMBOSS TOOLS
-acedb = galaxy.datatypes.data:Text
-asn1 = galaxy.datatypes.data:Text
-btwisted = galaxy.datatypes.data:Text
-cai = galaxy.datatypes.data:Text
-charge = galaxy.datatypes.data:Text
-checktrans = galaxy.datatypes.data:Text
-chips = galaxy.datatypes.data:Text
-clustal = galaxy.datatypes.data:Text
-codata = galaxy.datatypes.data:Text
-codcmp = galaxy.datatypes.data:Text
-coderet = galaxy.datatypes.data:Text
-compseq = galaxy.datatypes.data:Text
-cpgplot = galaxy.datatypes.data:Text
-cpgreport = galaxy.datatypes.data:Text
-cusp = galaxy.datatypes.data:Text
-cut = galaxy.datatypes.data:Text
-dan = galaxy.datatypes.data:Text
-dbmotif = galaxy.datatypes.data:Text
-diffseq = galaxy.datatypes.data:Text
-digest = galaxy.datatypes.data:Text
-dreg = galaxy.datatypes.data:Text
-einverted = galaxy.datatypes.data:Text
-embl = galaxy.datatypes.data:Text
-epestfind = galaxy.datatypes.data:Text
-equicktandem = galaxy.datatypes.data:Text
-est2genome = galaxy.datatypes.data:Text
-etandem = galaxy.datatypes.data:Text
-excel = galaxy.datatypes.data:Text
-feattable = galaxy.datatypes.data:Text
-fitch = galaxy.datatypes.data:Text
-freak = galaxy.datatypes.data:Text
-fuzznuc = galaxy.datatypes.data:Text
-fuzzpro = galaxy.datatypes.data:Text
-fuzztran = galaxy.datatypes.data:Text
-garnier = galaxy.datatypes.data:Text
-gcg = galaxy.datatypes.data:Text
-geecee = galaxy.datatypes.data:Text
-genbank = galaxy.datatypes.data:Text
-helixturnhelix = galaxy.datatypes.data:Text
-hennig86 = galaxy.datatypes.data:Text
-hmoment = galaxy.datatypes.data:Text
-ig = galaxy.datatypes.data:Text
-isochore = galaxy.datatypes.data:Text
-jackknifer = galaxy.datatypes.data:Text
-jackknifernon = galaxy.datatypes.data:Text
-markx10 = galaxy.datatypes.data:Text
-markx1 = galaxy.datatypes.data:Text
-markx0 = galaxy.datatypes.data:Text
-markx3 = galaxy.datatypes.data:Text
-markx2 = galaxy.datatypes.data:Text
-match = galaxy.datatypes.data:Text
-mega = galaxy.datatypes.data:Text
-meganon = galaxy.datatypes.data:Text
-motif = galaxy.datatypes.data:Text
-msf = galaxy.datatypes.data:Text
-nametable = galaxy.datatypes.data:Text
-ncbi = galaxy.datatypes.data:Text
-needle = galaxy.datatypes.data:Text
-newcpgreport = galaxy.datatypes.data:Text
-newcpgseek = galaxy.datatypes.data:Text
-nexus = galaxy.datatypes.data:Text
-nexusnon = galaxy.datatypes.data:Text
-noreturn = galaxy.datatypes.data:Text
-pair = galaxy.datatypes.data:Text
-palindrome = galaxy.datatypes.data:Text
-pepcoil = galaxy.datatypes.data:Text
-pepinfo = galaxy.datatypes.data:Text
-pepstats = galaxy.datatypes.data:Text
-phylip = galaxy.datatypes.data:Text
-phylipnon = galaxy.datatypes.data:Text
-pir = galaxy.datatypes.data:Text
-polydot = galaxy.datatypes.data:Text
-preg = galaxy.datatypes.data:Text
-prettyseq = galaxy.datatypes.data:Text
-primersearch = galaxy.datatypes.data:Text
-regions = galaxy.datatypes.data:Text
-score = galaxy.datatypes.data:Text
-selex = galaxy.datatypes.data:Text
-seqtable = galaxy.datatypes.data:Text
-showfeat = galaxy.datatypes.data:Text
-showorf = galaxy.datatypes.data:Text
-simple = galaxy.datatypes.data:Text
-sixpack = galaxy.datatypes.data:Text
-srs = galaxy.datatypes.data:Text
-srspair = galaxy.datatypes.data:Text
-staden = galaxy.datatypes.data:Text
-strider = galaxy.datatypes.data:Text
-supermatcher = galaxy.datatypes.data:Text
-swiss = galaxy.datatypes.data:Text
-syco = galaxy.datatypes.data:Text
-table = galaxy.datatypes.data:Text
-textsearch = galaxy.datatypes.data:Text
-vectorstrip = galaxy.datatypes.data:Text
-wobble = galaxy.datatypes.data:Text
-wordcount = galaxy.datatypes.data:Text
-tagseq = galaxy.datatypes.data:Text
-
-# ---- Data Type Sniff Order --------------------------------------------------
-
-[galaxy:sniff_order]
-
-05 = galaxy.datatypes.sequence:Maf
-10 = galaxy.datatypes.sequence:Lav
-15 = galaxy.datatypes.sequence:Fasta
-20 = galaxy.datatypes.sequence:Fastq
-25 = galaxy.datatypes.sequence:FastqSolexa
-30 = galaxy.datatypes.interval:Wiggle
-35 = galaxy.datatypes.images:Html
-40 = galaxy.datatypes.sequence:Axt
-45 = galaxy.datatypes.interval:Bed
-50 = galaxy.datatypes.interval:CustomTrack
-55 = galaxy.datatypes.interval:Gff
-60 = galaxy.datatypes.interval:Gff3
-65 = galaxy.datatypes.interval:Interval