From bb708af475f52d68e1963e29a7d1a2840f73b105 Mon Sep 17 00:00:00 2001 From: Ian Schenck Date: Fri, 13 Jul 2007 21:09:03 +0000 Subject: [PATCH] Metadata. Some of this is in early stages. There are definite needs to the interface, and there are some slight design issues here and there. Also, upload doesn't want to allow anything to remain Bed or Interval, probably because missing_meta is still there, and missing_meta should eventually go away altogether. Anyhow, I am unfortunately gone for the weekend. Please take note of any crashing the edit page gives you. --- lib/galaxy/datatypes/data.py | 13 +++- lib/galaxy/datatypes/interval.py | 25 ++++--- lib/galaxy/datatypes/metadata.py | 121 +++++++++++++++++++++++++------ lib/galaxy/datatypes/sequence.py | 6 +- lib/galaxy/datatypes/tabular.py | 31 ++++---- lib/galaxy/interfaces/root.py | 82 +++++++++------------ lib/galaxy/model/__init__.py | 23 ++++++ lib/galaxy/model/mapping.py | 2 +- templates/edit.tmpl | 104 -------------------------- templates/edit_data.tmpl | 79 ++++++++++++++++++++ 10 files changed, 283 insertions(+), 203 deletions(-) delete mode 100644 templates/edit.tmpl create mode 100644 templates/edit_data.tmpl diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index d51e1180c3e..bd4893d738f 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -13,6 +13,9 @@ class DataMeta( type ): """ def __init__( cls, name, bases, dict_ ): cls.metadata_spec = MetadataSpecCollection() + for base in bases: + if hasattr(base, "metadata_spec"): + cls.metadata_spec.update(base.metadata_spec) Statement.process( cls ) class Data( object ): @@ -26,7 +29,7 @@ class Data( object ): >>> DataTest.metadata_spec.test.name 'test' >>> DataTest.metadata_spec.test.desc - >>> DataTest.metadata_spec.test.attributes + 'test' >>> DataTest.metadata_spec.test.param @@ -155,6 +158,14 @@ class Data( object ): log.exception('Function %s is referred to in datatype %s for generating links for type %s, but is not accessible' % (self.supported_display_apps[type]['links_function'], self.__class__.__name__, type) ) return [] + def before_edit( self, dataset ): + """This function is called on the dataset before metadata is edited.""" + pass + + def after_edit( self, dataset ): + """This function is called on the dataset after metadata is edited.""" + pass + class Text( Data ): def write_from_stream(self, dataset, stream): diff --git a/lib/galaxy/datatypes/interval.py b/lib/galaxy/datatypes/interval.py index 478e75b5487..fd6b4632ff0 100644 --- a/lib/galaxy/datatypes/interval.py +++ b/lib/galaxy/datatypes/interval.py @@ -11,6 +11,7 @@ from galaxy import util from cgi import escape import urllib from bx.intervals.io import * +from galaxy.datatypes import metadata from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.tabular import Tabular @@ -37,10 +38,12 @@ class Interval( Tabular ): """Tab delimited data containing interval information""" """Add metadata elements""" - MetadataElement( name="chromCol" ) - MetadataElement( name="startCol" ) - MetadataElement( name="endCol" ) - MetadataElement( name="strandCol" ) + MetadataElement( name="chromCol", desc="Chrom column", param=metadata.ColumnParameter ) + MetadataElement( name="startCol", desc="Start column", param=metadata.ColumnParameter ) + MetadataElement( name="endCol", desc="End column", param=metadata.ColumnParameter ) + MetadataElement( name="strandCol", desc="Strand column", param=metadata.ColumnParameter, optional=True ) + MetadataElement( name="dbkey", desc="Database/Build", default="?", + param=metadata.SelectParameter, multiple=False, values=util.dbnames ) def __init__(self, **kwd): @@ -58,9 +61,6 @@ class Interval( Tabular ): def init_meta( self, dataset, copy_from=None ): Tabular.init_meta( self, dataset, copy_from=copy_from ) - for key in alias_spec: - setattr( dataset.metadata, key, '' ) - setattr( dataset.metadata, 'strandCol', '0' ) def set_peek( self, dataset ): """Set the peek and blurb text""" @@ -181,10 +181,13 @@ class Bed( Interval ): """Tab delimited data in BED format""" """Add metadata elements""" - MetadataElement( name="chromCol", default=1 ) - MetadataElement( name="startCol", default=2 ) - MetadataElement( name="endCol", default=3 ) - MetadataElement( name="strandCol", default=6 ) + """Add metadata elements""" + MetadataElement( name="chromCol", default=1, desc="Chrom column", param=metadata.ColumnParameter ) + MetadataElement( name="startCol", default=2, desc="Start column", param=metadata.ColumnParameter ) + MetadataElement( name="endCol", default=3, desc="End column", param=metadata.ColumnParameter ) + MetadataElement( name="strandCol", desc="Strand column", param=metadata.ColumnParameter, optional=True ) + MetadataElement( name="dbkey", desc="Database/Build", default=None, + param=metadata.SelectParameter, multiple=False, values=util.dbnames ) def missing_meta( self, dataset ): """Checks for empty meta values""" diff --git a/lib/galaxy/datatypes/metadata.py b/lib/galaxy/datatypes/metadata.py index 237aac033d5..4f0bf3147bd 100644 --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -1,5 +1,7 @@ import sys from cookbook.patterns import Bunch +from cookbook.odict import odict +from galaxy import form_builder # Taken in part from Elixir and how they do it: http://elixir.ematia.de @@ -23,75 +25,99 @@ class Statement( object ): for statement, args, kwargs in getattr( element, STATEMENTS, [] ): statement.target( element, *args, **kwargs ) -class MetadataSpecCollection( dict ): +class MetadataSpecCollection( odict ): ''' A simple extension of dict which allows cleaner access to items and allows the values to be iterated over directly as if it were a list. append() is also implemented for simplicity and does not "append". ''' + def __init__(self, dict = None): + odict.__init__(self, dict = None) def append( self, item ): self[item.name] = item def iter( self ): return self.itervalues() def __getattr__( self, name ): - return self[name] + return self.get(name) class MetadataParameter( object ): - def __init__( self, metadata, context ): + def __init__( self, spec, value, context ): ''' The "context" is simply the metadata collection/bunch holding - this piece of metadata. This is passed in to allow for + this piece of metadata. This is passed in to allow for metadata to validate against each other (note: this could turn - into a huge, recursive mess if not done with care). For + into a huge, recursive mess if not done with care). For example, a column assignment should validate against the number of columns in the dataset. ''' - self.metadata = metadata + self.spec = spec + self.value = value self.context = context - def marshal( self, value ): + + @classmethod + def marshal( cls, value ): ''' This method should/can be overridden to convert the incomming value to whatever type it is supposed to be. ''' return value - def validate( self, value ): + @classmethod + def validate( cls, value ): ''' Throw an exception if the value is invalid. ''' pass def get_html_field( self, value=None, other_values={} ): - raise TypeError("Abstract Method") - + return form_builder.TextField( self.spec.name, value=value or self.value ) + + def get_html( self ): + if self.spec.get("readonly"): + return self.value + if self.spec.get("optional"): + checked = False + if self.value: checked = "true" + checkbox = form_builder.CheckboxField( "is_" + self.spec.name, checked=checked ) + return checkbox.get_html() + self.get_html_field().get_html() + else: + return self.get_html_field().get_html() + @classmethod - def build_param( cls, element, context ): - return element.param( element, context ) - - + def unwrap( cls, form_value ): + value = cls.marshal(form_value) + cls.validate(value) + return value + class MetadataElementSpec( object ): ''' Defines a metadata element and adds it to the metadata_spec (which is a MetadataSpecCollection) of datatype. ''' - def __init__( self, datatype, name=None, desc=None, param=MetadataParameter, attributes=None, default=None ): + def __init__( self, datatype, name=None, desc=None, param=MetadataParameter, default=None, **kwargs ): self.name = name - self.desc = desc + self.desc = desc or name self.param = param - self.attributes = attributes self.default = default + # Catch-all, allows for extra attributes to be set + self.__dict__.update(kwargs) datatype.metadata_spec.append( self ) + def get( self, name ): + return self.__dict__.get(name, None) def hasAttribute( self, attribute ): return ((self.permission & attribute) == attribute) - def wrap( self, metadata ): - return self.param(metadata) - + def wrap( self, value, context ): + return self.param( self, value, context ) + def unwrap( self, form_value, context ): + return self.param.unwrap( form_value ) + # Basic attributes for describing metadata elements MetadataAttributes = Bunch( - READONLY = 1 + READONLY = 1, + OPTIONAL = 2 ) @@ -106,7 +132,8 @@ class MetadataCollection: def __init__(self, parent, spec): self.parent = parent self.bunch = parent._metadata or Bunch() - self.spec = spec or dict() + if spec is None: self.spec = MetadataSpecCollection() + else: self.spec = spec def __iter__(self): return self.bunch.__iter__() def get( self, key, default=None ): @@ -138,3 +165,53 @@ class MetadataCollection: MetadataElement = Statement(MetadataElementSpec) +""" +MetadataParameter sub-classes. +""" + +class SelectParameter( MetadataParameter ): + def __init__( self, spec, value, context ): + MetadataParameter.__init__( self, spec, value, context ) + self.values = spec.get("values") + def get_html_field( self, value=None, other_values={} ): + field = form_builder.SelectField( self.spec.name, + multiple=self.spec.get("multiple"), + display=self.spec.get("display") ) + for value, label in self.values: + try: + if value == self.value or value in self.value: + field.add_option( label, value, selected=True ) + else: + field.add_option( label, value, selected=False ) + except TypeError: + field.add_option( value, label, selected=False ) + + return field + + @classmethod + def marshal( cls, value ): + # split into a list, or return single value if list length = 1 + if len(value) == 1: return value[0] + return value + +class RangeParameter( SelectParameter ): + def __init__( self, spec, value, context ): + SelectParameter.__init__( self, spec, value, context ) + # The spec must be set with min and max values + _min = spec.get("min") or 1 + _max = spec.get("max") or 1 + step = self.spec.get("step") or 1 + self.values = zip(range( _min, _max, step ), range( _min, _max, step )) + + @classmethod + def marshal( cls, value ): + values = [int(x) for x in value] + if len(values) == 1: return values[0] + return values + +class ColumnParameter( RangeParameter ): + def __init__( self, spec, value, context ): + RangeParameter.__init__( self, spec, value, context ) + column_range = range( 1, context.metadata.columns+1, 1 ) + self.values = zip( column_range, column_range ) + diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 3c750ac70e7..fa4d75f1421 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -4,6 +4,9 @@ Image classes import data import logging +from galaxy.datatypes.metadata import MetadataElement +from galaxy.datatypes import metadata +from galaxy import util log = logging.getLogger(__name__) @@ -31,7 +34,8 @@ class Fasta( Sequence ): class Maf( Sequence ): """Class describing a Maf alignment""" - pass + MetadataElement( name="dbkey", desc="Database/Build", default="?", + param=metadata.SelectParameter, multiple=True, values=util.dbnames ) class Axt( Sequence ): """Class describing an axt alignment""" diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 9af1ba1c125..2ae2aebf56e 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -20,25 +20,10 @@ class Tabular( data.Text ): MetadataElement( name="columns", default=0, desc="Number of columns", - attributes=MetadataAttributes.READONLY ) + readonly=True ) def init_meta( self, dataset, copy_from=None ): data.Text.init_meta( self, dataset, copy_from=copy_from ) - try: - # This actually doesn't work at the moment. There needs - # to be hooks for the file corresponding to a dataset, so - # that when the file is written and closed, code can be - # run. That is idealy where this block would go. - maxcols = 0 - count = 0 - for line in open( dataset.file_name ): - count += 1 - if count > 1000: break - cols = len( line.split("\t") ) - if cols > maxcols: maxcols = cols - setattr( dataset.metadata, "columns", maxcols ) - except: - pass def missing_meta( self, dataset ): """Checks for empty meta values""" for key, value in dataset.metadata.items(): @@ -89,3 +74,17 @@ class Tabular( data.Text ): """Returns formated html of peek""" m_peek = self.make_html_table( dataset.peek ) return m_peek + + def before_edit( self, dataset ): + data.Text.before_edit( self, dataset ) + try: + maxcols = 0 + count = 0 + for line in open( dataset.file_name ): + count += 1 + if count > 1000: break + cols = len( line.split("\t") ) + if cols > maxcols: maxcols = cols + setattr( dataset.metadata, "columns", maxcols ) + except: + pass diff --git a/lib/galaxy/interfaces/root.py b/lib/galaxy/interfaces/root.py index 01937f5ce0f..37514143cdb 100644 --- a/lib/galaxy/interfaces/root.py +++ b/lib/galaxy/interfaces/root.py @@ -157,55 +157,43 @@ class Universe(common.Root): return self.index( trans ) p = util.Params(kwd, safe=False) - if p.edit_genome_btn: - err = None - # check for "valid" column assignments first - # Patch: don't validate these fields for mafs, etc. - # This will all go away soon - if isinstance( data.datatype, datatypes.interval.Tabular ): - for attr in 'chromCol', 'startCol', 'endCol', 'strandCol': - try: - num = int( getattr(p, attr) ) - if num < 0: raise Exception() - except: - if getattr(p, attr) != "" or attr != 'strandCol': - err = "Column assignments must be numbers greater than zero. Strand column may be ommitted (set to 0)." - if err: - trans.log_event( "Edit submitted bad values on dataset %s" % str(id) ) - return trans.fill_template( "edit.tmpl", data=data, dbnames=util.dbnames, err=err ) - - data.name = p.name - data.info = p.info - data.dbkey = p.dbkey - - # detect metadata changes, kind of ugly - changed = False - for attr in 'chromCol', 'startCol', 'endCol', 'strandCol': - try: - oldv = str(getattr(data.metadata, attr)).strip() or None - except: - oldv = None - newv = str(getattr(p, attr)).strip() or None - if oldv != newv: - changed = True - - if changed and isinstance( data.datatype, datatypes.interval.Tabular ): - data.metadata.chromCol = p.chromCol - data.metadata.startCol = p.startCol - data.metadata.endCol = p.endCol - data.metadata.strandCol = p.strandCol or '0' - data.mark_metadata_changed() - if data.missing_meta(): - data.extension = 'tabular' + + if p.change: + trans.app.datatypes_registry.change_datatype( data, p.datatype ) + trans.app.model.flush() + elif p.save: + for name, spec in data.datatype.metadata_spec.items(): + optional = p.get("is_"+name, None) + if optional and optional == 'true': + # optional element... + # == 'true' actually means it is NOT checked (and therefore ommitted) + setattr(data.metadata,name,None) else: - data.extension = 'interval' - - data.flush() - trans.log_event( "Completed editing of dataset id %s" % str(id) ) + setattr(data.metadata,name,spec.unwrap(p.get(name, None), p)) + + data.datatype.after_edit( data ) + trans.app.model.flush() + return trans.fill_template( "edit_complete.tmpl" ) - else: - trans.log_event( "Opened edit view on dataset %s" % str(id) ) - return trans.fill_template( "edit.tmpl", data=data, dbnames=util.dbnames, err=None ) + + data.datatype.before_edit( data ) + + if "dbkey" in data.datatype.metadata_spec and not data.metadata.dbkey: + # Copy dbkey into metadata, for backwards compatability + # This looks like it does nothing, but getting the dbkey + # returns the metadata dbkey unless it is None, in which + # case it resorts to the old dbkey. Setting the dbkey + # sets it properly in the metadata + data.metadata.dbkey = data.dbkey + metadata = list() + # a list of MetadataParemeters + for name, spec in data.datatype.metadata_spec.items(): + metadata.append( spec.wrap( data.metadata.get(name), + data ) ) + datatypes = [x for x in trans.app.datatypes_registry.datatypes_by_extension.iterkeys()] + trans.log_event( "Opened edit view on dataset %s" % str(id) ) + return trans.fill_template( "edit_data.tmpl", data=data, metadata=metadata, + datatypes=datatypes, err=None ) @web.expose def delete( self, trans, id = None, **kwd): diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 49e590cc906..5a8a78f291a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -194,6 +194,29 @@ class Dataset( object ): # Needs to accept a MetadataCollection, a bunch, or a dict self._metadata = Bunch( **dict( bunch.items() ) ) metadata = property( get_metadata, set_metadata ) + + + # This provide backwards compatibility with using the old dbkey + # field in the database. That field now maps to "old_dbkey" (see + # mapping.py) + def get_dbkey( self ): + try: + dbkey = self.metadata.dbkey[0] + except TypeError: + dbkey = self.metadata.dbkey + return dbkey or self.old_dbkey + def set_dbkey( self, value ): + if "dbkey" in self.datatype.metadata_spec: + if self.datatype.metadata_spec.dbkey.get("multiple"): + # Initialize a list if there isn't one + db_list = self.metadata.dbkey or list() + db_list[0] = value + self.metadata.dbkey = db_list + else: + self.metadata.dbkey = value + else: + self.old_dbkey = value + dbkey = property( get_dbkey, set_dbkey ) def change_datatype( self, new_ext ): datatypes_registry.change_datatype( self, new_ext ) diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 554cde9bd98..89d38511c3f 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -63,7 +63,7 @@ Dataset.table = Table( "dataset", metadata, Column( "blurb", TrimmedString( 255 ) ), Column( "peek" , TEXT ), Column( "extension", TrimmedString( 64 ) ), - Column( "dbkey", TrimmedString( 64 ) ), + Column( "dbkey", TrimmedString( 64 ), key="old_dbkey" ), # maps to old_dbkey, see __init__.py Column( "state", TrimmedString( 64 ) ), Column( "metadata", PickleType(), key="_metadata" ), Column( "parent_id", Integer, nullable=True ), diff --git a/templates/edit.tmpl b/templates/edit.tmpl deleted file mode 100644 index 83b3a736c31..00000000000 --- a/templates/edit.tmpl +++ /dev/null @@ -1,104 +0,0 @@ - - - - -Galaxy - - - - - - - - -
-
Editing: $data.name
-
-
- - - - - - - - - - - - - - - - - - - - -
Name:
Info:
Database: - -
- #if $data.missing_meta(): - - Check this box if the data contains intervals - #end if -
-
- - Chromosome: - Start: - End: - - - #set $strand = $data.metadata.get('strandCol', '') != '0' - - #if $strand - Strand: - #end if - - #if not $strand -

- Check this box if the data contains a strand column

- - #end if - #if $err -
$err
- #end if -

Note: The strand column is optional.

-
-
- -
-
-
- - - - - \ No newline at end of file diff --git a/templates/edit_data.tmpl b/templates/edit_data.tmpl new file mode 100644 index 00000000000..ccdded31b9e --- /dev/null +++ b/templates/edit_data.tmpl @@ -0,0 +1,79 @@ + + +#from galaxy.util.expressions import ExpressionContext + + + + +Galaxy + + + + + + +#if $getVar( 'error_message', None ) +
$error_message
+

+#end if + +#def do_metadata( $metadata, $data ) +#for $element in $metadata + $element.spec.desc + $element.get_html() + +#end for +#end def + +#def datatype( $dataset, $datatypes ) + +#end def + +
+
Edit Attributes
+
+
+ + + + + + + + +
Datatype:$datatype( $data, $datatypes )
+
+
+
+
+
+ + + + + $do_metadata( $metadata, $data ) + +
Name:
Info:
+ +
+
+
+
+ +
+
+ +
+
+ + + + \ No newline at end of file