opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-01 15:37:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add the ability to define datatypes in the configuration file.

Browse Source 
Datatypes are defined like:
[galaxy:datatypes]
bed = galaxy.datatypes.interval:Bed
png = galaxy.datatypes.images:Image,image/png

where the mime-type (default of text/plain) can be declared after the class name, as is seen with png.
This commit is contained in:
Daniel Blankenberg
2007-06-01 19:25:19 +00:00
parent 437399fc66
commit 4cbfce361b
17 changed files with 204 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/app.py
+7 -2
View File
@@ -1,7 +1,9 @@
import sys, os, atexit
from galaxy import config, db, jobs, util, tools, web
import galaxy.model
import galaxy.model.mapping
import galaxy.datatypes.registry
from galaxy.interfaces import root, tool_runner, proxy, async, admin, user, error, dataset
from galaxy.web import middleware
@@ -13,6 +15,9 @@ class UniverseApplication( object ):
self.config = config.Configuration( **kwargs )
self.config.check()
config.configure_logging( self.config )
#Set up datatypes registry
self.datatypes_registry = galaxy.datatypes.registry.Registry(datatypes = self.config.datatypes)
galaxy.model.set_datatypes_registry(self.datatypes_registry)
# Connect up the object model
if self.config.database_connection:
self.model = galaxy.model.mapping.init( self.config.file_path,
@@ -23,7 +28,7 @@ class UniverseApplication( object ):
"sqlite://%s?isolation_level=IMMEDIATE" % self.config.database,
create_tables = True )
# Initialize the tools
self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path )
self.toolbox = tools.ToolBox( self.config.tool_config, self.config.tool_path, datatypes_registry = self.datatypes_registry )
# Start the job queue
self.job_queue = jobs.JobQueue( self )
self.heartbeat = None
@@ -46,7 +51,7 @@ def app_factory( global_conf, **kwargs ):
if 'app' in kwargs:
app = kwargs.pop( 'app' )
else:
app = UniverseApplication( **kwargs )
app = UniverseApplication( global_conf = global_conf, **kwargs )
atexit.register( app.shutdown )
# Create the universe WSGI application
webapp = web.framework.WebApplication()
lib/galaxy/config.py
+11
View File
@@ -5,6 +5,7 @@ Universe configuration builder.
import sys, os
import logging, logging.config
from optparse import OptionParser
import ConfigParser
log = logging.getLogger( __name__ )
@@ -45,6 +46,16 @@ class Configuration( object ):
self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "" )
self.use_heartbeat = kwargs.get( 'use_heartbeat', False )
self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",")
#Parse global_conf
global_conf = kwargs.get( 'global_conf', None )
global_conf_parser = ConfigParser.ConfigParser()
if global_conf and "__file__" in global_conf:
global_conf_parser.read(global_conf['__file__'])
#Store datatypes
try:
self.datatypes = global_conf_parser.items("galaxy:datatypes")
except ConfigParser.NoSectionError:
self.datatypes = []
def get( self, key, default ):
return self.config_dict.get( key, default )
def check( self ):
lib/galaxy/datatypes/__init__.py
-57
View File
@@ -1,57 +0,0 @@
"""
Contains data definitions
"""
import logging
import data, interval, images, sequence
from cookbook.patterns import Bunch
log = logging.getLogger(__name__)
datatypes_by_extension = {
'data' : data.Data(),
'bed' : interval.Bed(),
'txt' : data.Text(),
'text' : data.Text(),
'interval' : interval.Interval(),
'tabular' : interval.Tabular(),
'png' : images.Image(),
'pdf' : images.Image(),
'fasta' : sequence.Fasta(),
'maf' : sequence.Maf(),
'axt' : sequence.Axt(),
'gff' : interval.Gff(),
'wig' : interval.Wiggle(),
'gmaj.zip' : images.Gmaj(),
'laj' : images.Laj(),
'lav' : sequence.Lav(),
'html' : images.Html(),
'customtrack' : interval.CustomTrack()
}
def get_datatype_by_extension( ext ):
"""
Returns a datatype based on an extension
"""
try:
builder = datatypes_by_extension[ext]
except KeyError:
builder = data.Text()
log.warning('unkown extension in data factory %s' % ext)
return builder
def change_datatype( data, ext ):
data.extension = ext
data.init_meta()
if data.has_data():
data.set_peek()
return data
def old_change_datatype(data, ext):
"""
Creates and returns a new datatype based on an existing data and an extension
"""
newdata = factory(ext)(id=data.id)
for key, value in data.__dict__.items():
setattr(newdata, key, value)
newdata.ext = ext
return newdata
lib/galaxy/datatypes/data.py
+2 -7
View File
@@ -1,6 +1,7 @@
import logging, os, sys, time, sets, tempfile
from galaxy import util
from cgi import escape
import galaxy.datatypes.registry
log = logging.getLogger(__name__)
@@ -87,13 +88,7 @@ class Text( Data ):
def get_mime(self):
"""Returns the mime type of the data"""
try:
ext = self.ext.lower()
if ext in util.text_types:
return 'text/plain'
return util.mime_types[ext]
except KeyError:
return 'application/octet-stream'
return galaxy.datatypes.registry.Registry().get_mimetype_by_extension( self.extension.lower() )
def set_peek(self, dataset):
dataset.peek = get_file_peek( dataset.file_name )
lib/galaxy/datatypes/registry.py
+110
View File
@@ -0,0 +1,110 @@
"""
Provides mapping between extensions and datatypes, mime-types, etc.
"""
import logging
import data, interval, images, sequence
class Registry( object ):
def __init__( self, datatypes = [] ):
self.log = logging.getLogger(__name__)
self.datatypes_by_extension = {}
self.mimetypes_by_extension = {}
for ext, kind in datatypes:
try:
mime_type = 'text/plain' #default type of plain text
fields = kind.split(",")
if len(fields)>1:
kind = fields[0].strip()
mime_type = fields[1].strip()
fields = kind.split(":")
datatype_module = fields[0]
datatype_class = fields[1]
fields = datatype_module.split(".")
module = __import__(fields.pop(0))
for mod in fields: module = getattr(module,mod)
self.datatypes_by_extension[ext] = getattr(module, datatype_class)()
self.mimetypes_by_extension[ext] = mime_type
except:
self.log.warning('error loading datatype: %s' % ext)
#default values
if len(self.datatypes_by_extension) < 1:
self.datatypes_by_extension = {
'data' : data.Data(),
'bed' : interval.Bed(),
'txt' : data.Text(),
'text' : data.Text(),
'interval' : interval.Interval(),
'tabular' : interval.Tabular(),
'png' : images.Image(),
'pdf' : images.Image(),
'fasta' : sequence.Fasta(),
'maf' : sequence.Maf(),
'axt' : sequence.Axt(),
'gff' : interval.Gff(),
'wig' : interval.Wiggle(),
'gmaj.zip' : images.Gmaj(),
'laj' : images.Laj(),
'lav' : sequence.Lav(),
'html' : images.Html(),
'customtrack' : interval.CustomTrack()
}
self.mimetypes_by_extension = {
'data' : 'application/octet-stream',
'bed' : 'text/plain',
'txt' : 'text/plain',
'text' : 'text/plain',
'interval' : 'text/plain',
'tabular' : 'text/plain',
'png' : 'image/png',
'pdf' : 'application/pdf',
'fasta' : 'text/plain',
'maf' : 'text/plain',
'axt' : 'text/plain',
'gff' : 'text/plain',
'wig' : 'text/plain',
'gmaj.zip' : 'application/zip',
'laj' : 'text/plain',
'lav' : 'text/plain',
'html' : 'text/html',
'customtrack' : 'text/plain'
}
def get_mimetype_by_extension(self, ext ):
"""
Returns a mimetype based on an extension
"""
try:
mimetype = self.mimetypes_by_extension[ext]
except KeyError:
#datatype was never declared
mimetype = 'application/octet-stream'
self.log.warning('unkown mimetype in data factory %s' % ext)
return mimetype
def get_datatype_by_extension(self, ext ):
"""
Returns a datatype based on an extension
"""
try:
builder = self.datatypes_by_extension[ext]
except KeyError:
builder = data.Text()
self.log.warning('unkown extension in data factory %s' % ext)
return builder
def change_datatype(self, data, ext ):
data.extension = ext
data.init_meta()
if data.has_data():
data.set_peek()
return data
def old_change_datatype(self, data, ext):
"""
Creates and returns a new datatype based on an existing data and an extension
"""
newdata = factory(ext)(id=data.id)
for key, value in data.__dict__.items():
setattr(newdata, key, value)
newdata.ext = ext
return newdata
lib/galaxy/interfaces/root.py
+11 -11
View File
@@ -97,7 +97,7 @@ class Universe(common.Root):
data = self.app.model.Dataset.get( id )
if data:
mime = data.get_mime()
mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() )
trans.response.set_content_type(mime)
if tofile:
fStat = os.stat(data.file_name)
@@ -119,7 +119,7 @@ class Universe(common.Root):
data = self.app.model.Dataset.get( id )
if data:
if isinstance(data.datatype, datatypes.interval.Interval) or isinstance(data.datatype, datatypes.interval.CustomTrack):
mime = data.get_mime()
mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() )
trans.response.set_content_type(mime)
file_name = data.as_bedfile()
trans.log_event( "Display dataset id %s as BED" % str(id) )
@@ -329,7 +329,7 @@ class Universe(common.Root):
send_to_err = "You can't send histories to yourself"
else:
for history in histories:
new_history = self.copy_history(history)
new_history = self.copy_history(history, trans)
new_history.name = history.name+" from "+user.email
new_history.user_id = send_to_user.id
"""
@@ -362,7 +362,7 @@ class Universe(common.Root):
if user:
if import_history.user_id == user.id:
return trans.show_error_message( "You cannot import your own history.")
new_history = self.copy_history(import_history)
new_history = self.copy_history(import_history, trans)
new_history.name = "imported: "+new_history.name
new_history.user_id = user.id
new_history.add_galaxy_session(trans.get_galaxy_session( create=True ))
@@ -372,7 +372,7 @@ class Universe(common.Root):
trans.log_event( "History imported, id: %s, name: '%s': " % (str(new_history.id) , new_history.name ) )
return trans.fill_template("history_imported.tmpl", history=new_history)
elif not user_history.datasets or confirm:
new_history = self.copy_history(import_history)
new_history = self.copy_history(import_history, trans)
new_history.name = "imported: "+new_history.name
new_history.user_id = None
new_history.add_galaxy_session(trans.get_galaxy_session( create=True ))
@@ -497,7 +497,7 @@ class Universe(common.Root):
"""Copies a dataset and makes primary"""
try:
old_data = self.app.model.Dataset.get( id )
new_data = self.copy_dataset(old_data)
new_data = self.copy_dataset(old_data, trans)
## new_data.parent = None
## history = trans.app.model.History.get( old_data.history_id )
history = trans.get_history()
@@ -551,10 +551,10 @@ class Universe(common.Root):
# ---- Work methods -----------------------------------------------------
def copy_dataset(self, src, parent_id=None):
def copy_dataset(self, src, trans, parent_id=None):
des = self.app.model.Dataset()
des.flush()
des = datatypes.change_datatype(des, src.ext)
des = trans.app.dataset_registry.change_datatype(des, src.ext)
des.name = src.name
des.info = src.info
des.blurb = src.blurb
@@ -571,18 +571,18 @@ class Universe(common.Root):
des.flush()
return des
def copy_history(self, src):
def copy_history(self, src, trans):
des = self.app.model.History()
des.flush()
des.name = src.name
des.user_id = src.user_id
for data in src.datasets:
new_data = self.copy_dataset(data)
new_data = self.copy_dataset(data, trans)
des.add_dataset(new_data)
new_data.hid = data.hid
new_data.flush()
for child_assoc in data.children:
new_child = self.copy_dataset(child_assoc.child)
new_child = self.copy_dataset(child_assoc.child, trans)
new_assoc = self.app.model.DatasetAssociation( child.designation )
new_assoc.child = new_child
new_assoc.parent = new_data
lib/galaxy/model/__init__.py
+11 -8
View File
@@ -11,6 +11,15 @@ import galaxy.datatypes
from cookbook.patterns import Bunch
from galaxy import util
import tempfile
import galaxy.datatypes.registry
datatypes_registry = galaxy.datatypes.registry.Registry()
def set_datatypes_registry( d_registry ):
"""
Set up datatypes_registry
"""
datatypes_registry = d_registry
class User( object ):
def __init__( self, email=None, password=None ):
@@ -169,7 +178,7 @@ class Dataset( object ):
return os.path.join( self.file_path, "dataset_%d.dat" % self.id )
@property
def datatype( self ):
return galaxy.datatypes.get_datatype_by_extension( self.extension )
return datatypes_registry.get_datatype_by_extension( self.extension )
def get_size( self ):
"""
Returns the size of the data on disk
@@ -218,13 +227,7 @@ class Dataset( object ):
os.remove( temp_name )
def get_mime(self):
"""Returns the mime type of the data"""
try:
ext = self.extension.lower()
if ext in util.text_types:
return 'text/plain'
return util.mime_types[ext]
except KeyError:
return 'application/octet-stream'
return datatypes_registry.get_mimetype_by_extension( self.extension.lower() )
def set_peek( self ):
return self.datatype.set_peek( self )
def init_meta( self ):
lib/galaxy/tools/__init__.py
+7 -4
View File
@@ -21,6 +21,7 @@ from grouping import *
from galaxy.util.expressions import ExpressionContext
from galaxy.tools.test import ToolTestBuilder
from galaxy.tools.actions import DefaultToolAction
import galaxy.datatypes.registry
log = logging.getLogger( __name__ )
@@ -32,7 +33,7 @@ class ToolBox( object ):
Container for a collection of tools
"""
def __init__( self, config_filename, tool_root_dir ):
def __init__( self, config_filename, tool_root_dir, datatypes_registry = galaxy.datatypes.registry.Registry() ):
"""
Create a toolbox from the config file names by `config_filename`,
using `tool_root_directory` as the base directory for finding
@@ -42,6 +43,7 @@ class ToolBox( object ):
self.tools_and_sections_by_id = {}
self.sections = []
self.tool_root_dir = tool_root_dir
self.datatypes_registry = datatypes_registry
try:
self.init_tools( config_filename )
except:
@@ -86,7 +88,7 @@ class ToolBox( object ):
ToolClass = getattr( mod, cls )
else:
ToolClass = Tool
return ToolClass( config_file, root )
return ToolClass( config_file, root, datatypes_registry = self.datatypes_registry )
def reload( self, tool_id ):
"""
@@ -163,13 +165,14 @@ class Tool:
"""
Represents a computational tool that can be executed through Galaxy.
"""
def __init__( self, config_file, root ):
def __init__( self, config_file, root, datatypes_registry = galaxy.datatypes.registry.Registry() ):
"""
Load a tool from the config named by `config_file`
"""
# Determine the full path of the directory where the tool config is
self.config_file = config_file
self.tool_dir = os.path.dirname( config_file )
self.datatypes_registry = datatypes_registry
# Parse XML element containing configuration
self.parse( root )
@@ -432,7 +435,7 @@ class Tool:
Also, if the parameter has a 'required_enctype' add it to the set
enctypes.
"""
param = ToolParameter.build( self, input_elem )
param = ToolParameter.build( self, input_elem, datatypes_registry=self.datatypes_registry )
param_enctype = param.get_required_enctype()
if param_enctype:
enctypes.add( param_enctype )
lib/galaxy/tools/parameters.py
+10 -5
View File
@@ -3,7 +3,8 @@ Classes encapsulating tool parameters
"""
import logging, string, sys
from galaxy import config, datatypes, util, form_builder
from galaxy import config, datatypes, util, form_builder
import galaxy.datatypes.registry
import validation
from elementtree.ElementTree import XML, Element
@@ -105,13 +106,17 @@ class ToolParameter( object ):
validator.validate( value, history )
@classmethod
def build( cls, tool, param ):
def build( cls, tool, param, datatypes_registry = galaxy.datatypes.registry.Registry() ):
"""Factory method to create parameter of correct type"""
param_type = param.get("type")
if not param_type or param_type not in parameter_types:
raise ValueError( "Unknown tool parameter type '%s'" % param_type )
else:
return parameter_types[param_type]( tool, param )
#data parameter requires datatypes_registry
if param_type in ['data']:
return parameter_types[param_type]( tool, param, datatypes_registry = datatypes_registry )
else:
return parameter_types[param_type]( tool, param )
class TextToolParameter( ToolParameter ):
"""
@@ -536,13 +541,13 @@ class DataToolParameter( ToolParameter ):
<option value="5" selected>5: Unnamed dataset</option>
</select>
"""
def __init__( self, tool, elem ):
def __init__( self, tool, elem, datatypes_registry = galaxy.datatypes.registry.Registry() ):
ToolParameter.__init__( self, tool, elem )
# Build tuple of classes for supported data formats
formats = []
extensions = elem.get( 'format', 'data' ).split( "," )
for extension in extensions:
formats.append( datatypes.get_datatype_by_extension( extension.lower() ).__class__ )
formats.append( datatypes_registry.get_datatype_by_extension( extension.lower() ).__class__ )
self.formats = tuple( formats )
self.multiple = str_bool( elem.get( 'multiple', False ) )
self.optional = str_bool( elem.get( 'optional', False ) )
tools/data_source/biomart_filter.py
+1 -1
View File
@@ -11,7 +11,7 @@ def exec_before_job( app, inp_data, out_data, param_dict, tool=None):
data_type = param_dict.get( 'type', 'text' )
if data_type == 'text': data_type='interval' #All data from biomart is TSV, assume interval
name, data = out_data.items()[0]
data = datatypes.change_datatype(data, data_type)
data = app.datatypes_registry.change_datatype(data, data_type)
data.name = data_name
out_data[name] = data
tools/data_source/encode_import_code.py
+1 -1
View File
@@ -117,7 +117,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
data.name = data.name + " (" + description + ")"
data.dbkey = dbkey
data.info = data.name
datatypes.change_datatype( data, file_type )
data = app.datatypes_registry.change_datatype( data, file_type )
data.init_meta()
data.set_peek()
app.model.flush()
tools/data_source/hbvar_filter.py
+2 -2
View File
@@ -10,7 +10,7 @@ def exec_before_job( app, inp_data, out_data, param_dict, tool=None):
data_type = param_dict.get( 'type', 'text' )
if data_type == 'text': data_type='interval' #All data is TSV, assume interval
name, data = out_data.items()[0]
data = datatypes.change_datatype(data, data_type)
data = app.datatypes_registry.change_datatype(data, data_type)
data.name = data_name
out_data[name] = data
@@ -70,6 +70,6 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=No
shutil.move(temp_filename,data.file_name)
else:
data = datatypes.change_datatype(data, 'tabular')
data = app.datatypes_registry.change_datatype(data, 'tabular')
data.set_peek()
data.flush()
tools/data_source/microbial_import_code.py
+1 -1
View File
@@ -225,7 +225,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
data.name = data.name + " (" + microbe_info[kingdom][org]['chrs'][chr]['data'][description]['feature'] +" for "+microbe_info[kingdom][org]['name']+":"+chr + ")"
data.dbkey = dbkey
data.info = data.name
datatypes.change_datatype( data, file_type )
data = app.datatypes_registry.change_datatype( data, file_type )
data.init_meta()
data.set_peek()
app.model.flush()
tools/data_source/ucsc_tablebrowser_code.py
+3 -4
View File
@@ -21,9 +21,8 @@ def exec_before_job( app, inp_data, out_data, param_dict, tool=None):
ext = outputType
try: ext = outputType_to_ext[outputType]
except: pass
if ext not in datatypes.datatypes_by_extension: ext = 'interval'
data = datatypes.change_datatype(data, ext)
if ext not in app.datatypes_registry.datatypes_by_extension: ext = 'interval'
data = app.datatypes_registry.change_datatype(data, ext)
#store ucsc parameters temporarily in output file
out = open(data.file_name,'w')
@@ -40,6 +39,6 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=No
if not isinstance(data.datatype, datatypes.interval.Bed) and isinstance(data.datatype, datatypes.interval.Interval):
data.set_meta()
if data.missing_meta(): data = datatypes.change_datatype(data, 'tabular')
if data.missing_meta(): data = app.datatypes_registry.change_datatype(data, 'tabular')
data.set_peek()
data.flush()
tools/emboss/emboss_format_corrector.py
+4 -4
View File
@@ -1,7 +1,7 @@
#EMBOSS format corrector
import operator
from galaxy import datatypes
#from galaxy import datatypes
#Properly set file formats after job run
def exec_after_process( app, inp_data, out_data, param_dict,tool, stdout, stderr):
@@ -25,7 +25,7 @@ def exec_after_process( app, inp_data, out_data, param_dict,tool, stdout, stderr
outputType = "Tabular"
elif outputType == 'text':
outputType = "txt"
data = datatypes.change_datatype(data, outputType)
data = app.datatypes_registry.change_datatype(data, outputType)
data.flush()
data_count+=1
@@ -35,7 +35,7 @@ def exec_after_process( app, inp_data, out_data, param_dict,tool, stdout, stderr
wants_plot = param_dict.get( 'html_out'+str(data_count), None )
ext = "html"
if wants_plot == "yes":
data = datatypes.change_datatype(data, ext)
data = app.datatypes_registry.change_datatype(data, ext)
data.flush()
data_count+=1
@@ -45,6 +45,6 @@ def exec_after_process( app, inp_data, out_data, param_dict,tool, stdout, stderr
wants_plot = param_dict.get( 'plot'+str(data_count), None )
ext = "png"
if wants_plot == "yes":
data = datatypes.change_datatype(data, ext)
data = app.datatypes_registry.change_datatype(data, ext)
data.flush()
data_count+=1
tools/filters/pasteWrapper_code.py
+1 -1
View File
@@ -3,5 +3,5 @@ from galaxy import datatypes
def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr):
for name, data in out_data.items():
if data.ext == "bed":
data = datatypes.change_datatype(data, "interval")
data = app.datatypes_registry.change_datatype(data, "interval")
data.flush()
universe_wsgi.ini.sample
+22
View File
@@ -105,3 +105,25 @@ cache_seconds = 360
document_root = %(here)s/static/light_hatched_style/blue
#document_root = %(here)s/static/light_hatched_style/green
#document_root = %(here)s/static/old_blue_style
[galaxy:datatypes]
data = galaxy.datatypes.data:Data,application/octet-stream
bed = galaxy.datatypes.interval:Bed
txt = galaxy.datatypes.data:Text
text = galaxy.datatypes.data:Text
interval = galaxy.datatypes.interval:Interval
tabular = galaxy.datatypes.interval:Tabular
png = galaxy.datatypes.images:Image,image/png
pdf = galaxy.datatypes.images:Image,application/pdf
gif = galaxy.datatypes.images:Image,image/gif
jpg = galaxy.datatypes.images:Image,image/jpeg
fasta = galaxy.datatypes.sequence:Fasta
maf = galaxy.datatypes.sequence:Maf
axt = galaxy.datatypes.sequence:Axt
gff = galaxy.datatypes.interval:Gff
wig = galaxy.datatypes.interval:Wiggle
gmaj.zip = galaxy.datatypes.images:Gmaj,application/zip
laj = galaxy.datatypes.images:Laj
lav = galaxy.datatypes.sequence:Lav
html = galaxy.datatypes.images:Html,text/html
customtrack = galaxy.datatypes.interval:CustomTrack