diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 545afd49cd9..93a8e8adfcb 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -9,7 +9,6 @@ log = logging.getLogger(__name__) # Valid first column and strand column values vor bed, other formats col1_startswith = ['chr', 'chl', 'groupun', 'reftig_', 'scaffold', 'super_', 'vcho'] valid_strand = ['+', '-', '.'] -gzip_magic = '\037\213' class DataMeta( type ): """ @@ -86,10 +85,19 @@ class Data( object ): def set_readonly_meta( self, dataset ): """Unimplemented method, resets the readonly metadata values""" return True - def missing_meta( self, dataset ): - """Checks for empty metadata values, Returns True if non-optional metadata is missing""" - for key, value in dataset.metadata.items(): - if dataset.metadata.spec[key].get("optional"): continue #we skip check for optional values here + def missing_meta( self, dataset, check = [], skip = [] ): + """ + Checks for empty metadata values, Returns True if non-optional metadata is missing + Specifying a list of 'check' values will only check those names provided; when used, optionality is ignored + Specifying a list of 'skip' items will return True even when a named metadata value is missing + """ + if check: + to_check = [ ( to_check, dataset.metadata.get( to_check ) ) for to_check in check ] + else: + to_check = dataset.metadata.items() + for key, value in to_check: + if key in skip or ( not check and dataset.metadata.spec[key].get( "optional" ) ): + continue #we skip check for optional and nonrequested values here if not value: return True return False @@ -328,7 +336,7 @@ def get_file_peek( file_name, WIDTH=256, LINE_COUNT=5 ): line = line[ :WIDTH ] if not data_checked and line: data_checked = True - if line[0:2] == gzip_magic: + if line[0:2] == util.gzip_magic: file_type = 'gzipped' break else: diff --git a/lib/galaxy/datatypes/metadata.py b/lib/galaxy/datatypes/metadata.py index 7a972f24e76..90bd28d0f48 100644 --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -1,8 +1,9 @@ -import sys, logging +import sys, logging, copy, shutil from galaxy.util import string_as_bool from galaxy.util.odict import odict from galaxy.web import form_builder +import galaxy.model log = logging.getLogger( __name__ ) @@ -75,7 +76,13 @@ class MetadataCollection: def get_html_by_name( self, name, **kwd ): if name in self.spec: return self.spec[name].param.get_html( value=getattr( self, name ), context=self, **kwd ) - + def make_dict_copy( self, to_copy ): + """Makes a deep copy of input iterable to_copy according to self.spec""" + rval = {} + for key, value in to_copy.items(): + if key in self.spec: + rval[key] = self.spec[key].param.make_copy( value, target_context=self, source_context=to_copy ) + return rval class MetadataSpecCollection( odict ): """ @@ -121,7 +128,10 @@ class MetadataParameter( object ): def to_string( self, value ): return str( value ) - + + def make_copy( self, value, target_context = None, source_context = None ): + return copy.deepcopy( value ) + @classmethod def marshal ( cls, value ): """ @@ -151,7 +161,6 @@ class MetadataParameter( object ): """ return value - class MetadataElementSpec( object ): """ Defines a metadata element and adds it to the metadata_spec (which @@ -280,16 +289,14 @@ class ColumnTypesParameter( MetadataParameter ): return ",".join( map( str, value ) ) class PythonObjectParameter( MetadataParameter ): - def __init__( self, spec ): - MetadataParameter.__init__( self, spec ) def to_string( self, value ): if not value: - return self.spec.to_string( self.spec.no_value ) - return self.spec.to_string( value ) + return self.spec._to_string( self.spec.no_value ) + return self.spec._to_string( value ) def get_html_field( self, value=None, context={}, other_values={}, **kwd ): - return form_builder.TextField( self.spec.name, value=self.to_string( value ) ) + return form_builder.TextField( self.spec.name, value=self._to_string( value ) ) def get_html( self, value=None, context={}, other_values={}, **kwd ): return str( self ) @@ -297,3 +304,40 @@ class PythonObjectParameter( MetadataParameter ): @classmethod def marshal( cls, value ): return value + +class FileParameter( MetadataParameter ): + + def to_string( self, value ): + if not value: + return str( self.spec.no_value ) + return value.file_name + + def get_html_field( self, value=None, context={}, other_values={}, **kwd ): + return form_builder.TextField( self.spec.name, value=str( value.id ) ) + + def get_html( self, value=None, context={}, other_values={}, **kwd ): + return "
No display available for Metadata Files
" + + def wrap( self, value ): + if isinstance( value, galaxy.model.MetadataFile ): + return value + try: + return galaxy.model.MetadataFile.get( value ) + except: + #value was not a valid id + return None + + def make_copy( self, value, target_context = None, source_context = None ): + value = self.wrap( value ) + if value: + new_value = galaxy.model.MetadataFile( dataset = target_context.parent, name = self.spec.name ) + new_value.flush() + shutil.copy( value.file_name, new_value.file_name ) + return self.unwrap( new_value ) + return None + + @classmethod + def marshal( cls, value ): + if isinstance( value, galaxy.model.MetadataFile ): + value = value.id + return value diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 692b257f7b1..0d90ad72f38 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -8,6 +8,7 @@ import re from cgi import escape from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes import metadata +import galaxy.model from galaxy import util from sniff import * @@ -24,7 +25,6 @@ class Alignment( Sequence ): """Add metadata elements""" MetadataElement( name="species", desc="Species", default=[], param=metadata.SelectParameter, multiple=True, readonly=True, no_value=None ) - MetadataElement( name="species_chromosomes", desc="Species Chromosomes", value={}, param=metadata.PythonObjectParameter, readonly=True, no_value={}, to_string=str, visible=False ) class Fasta( Sequence ): """Class representing a FASTA sequence""" @@ -192,23 +192,30 @@ except: class Maf( Alignment ): """Class describing a Maf alignment""" file_ext = "maf" + + #Readonly and optional, users can't unset it, but if it is not set, we are generally ok; if required use a metadata validator in the tool definition + MetadataElement( name="species_chromosomes", desc="Species Chromosomes", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) + MetadataElement( name="maf_index", desc="MAF Index File", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True ) def init_meta( self, dataset, copy_from=None ): Alignment.init_meta( self, dataset, copy_from=copy_from ) def set_meta( self, dataset, overwrite = True, **kwd ): """ - Parses and sets species and chromosomes from MAF files. + Parses and sets species, chromosomes, index from MAF file. """ + #these metadata values are not accessable by users, always overwrite + species = [] species_chromosomes = {} + maf_reader = bx.align.maf.Reader( open( dataset.file_name ) ) + indexes = bx.interval_index_file.Indexes() try: - for i, m in enumerate( bx.align.maf.Reader( open(dataset.file_name) ) ): - for c in m.components: - ## spec,chrom = bx.align.maf.src_split( c.src ) - ## if not spec or not chrom: spec = chrom = c.src - # "src_split" finds the rightmost dot, which is probably - # wrong in general, and certainly here. + while True: + pos = maf_reader.file.tell() + block = maf_reader.next() + if block is None: break + for c in block.components: spec = c.src chrom = None if "." in spec: @@ -218,20 +225,44 @@ class Maf( Alignment ): species_chromosomes[spec] = [] if chrom and chrom not in species_chromosomes[spec]: species_chromosomes[spec].append( chrom ) - # only check first 100,000 blocks for species - if i > 100000: break - except: + indexes.add( c.src, c.forward_strand_start, c.forward_strand_end, pos, max=c.src_size ) + except: #bad MAF file pass - #these metadata values are not accessable by users, always overwrite dataset.metadata.species = species - dataset.metadata.species_chromosomes = species_chromosomes + #only overwrite the contents if our newly determined chromosomes don't match stored + chrom_file = dataset.metadata.species_chromosomes + compare_chroms = {} + if chrom_file: + try: + for line in open( chrom_file.file_name ): + fields = line.split( "\t" ) + if fields: + spec = fields.pop( 0 ) + if spec: + compare_chroms[spec] = fields + except: + pass + #write out species chromosomes again only if values are different + if not species_chromosomes or compare_chroms != species_chromosomes: + tmp_file = tempfile.TemporaryFile( 'w+b' ) + for spec, chroms in species_chromosomes.items(): + tmp_file.write( "%s\t%s\n" % ( spec, "\t".join( chroms ) ) ) + + if not chrom_file: + chrom_file = galaxy.model.MetadataFile( dataset = dataset, name = "species_chromosomes" ) + chrom_file.flush() + tmp_file.seek( 0 ) + open( chrom_file.file_name, 'wb' ).write( tmp_file.read() ) + dataset.metadata.species_chromosomes = chrom_file + tmp_file.close() + + index_file = dataset.metadata.maf_index + if not index_file: + index_file = galaxy.model.MetadataFile( dataset = dataset, name="maf_index" ) + index_file.flush() + indexes.write( open( index_file.file_name, 'w' ) ) + dataset.metadata.maf_index = index_file - def missing_meta( self, dataset ): - """Checks to see if species is set""" - if dataset.metadata.species in [None, []]: - return True - return False - def display_peek( self, dataset ): """Returns formated html of peek""" return self.make_html_table( dataset ) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index a7504058a9e..02853b3256f 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -475,7 +475,16 @@ class JobWrapper( object ): def get_input_fnames( self ): job = model.Job.get( self.job_id ) - return [ da.dataset.file_name for da in job.input_datasets if da.dataset ] + filenames = [] + for da in job.input_datasets: #da is JobToInputDatasetAssociation object + if da.dataset: + filenames.append( da.dataset.file_name ) + #we will need to stage in metadata file names also + #TODO: would be better to only stage in metadata files that are actually needed (found in command line, referenced in config files, etc.) + for key, value in da.dataset.metadata.items(): + if isinstance( value, model.MetadataFile ): + filenames.append( value.file_name ) + return filenames def get_output_fnames( self ): job = model.Job.get( self.job_id ) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 3fcc3d857c1..de157193e13 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -5,7 +5,7 @@ Naming: try to use class names that have a distinct plural form so that the relationship cardinalities are obvious (e.g. prefer Dataset to Data) """ -import os.path, os, errno, copy +import os.path, os, errno import sha import galaxy.datatypes from galaxy.util.bunch import Bunch @@ -165,7 +165,7 @@ class HistoryDatasetAssociation( object ): return self._metadata_collection def set_metadata( self, bunch ): # Needs to accept a MetadataCollection, a bunch, or a dict - self._metadata = dict( [ ( key, copy.deepcopy( value ) ) for key, value in bunch.items() ] ) + self._metadata = self.metadata.make_dict_copy( bunch ) metadata = property( get_metadata, set_metadata ) """ @@ -223,8 +223,8 @@ class HistoryDatasetAssociation( object ): return self.datatype.set_meta( self, **kwd ) def set_readonly_meta( self, **kwd ): return self.datatype.set_readonly_meta( self, **kwd ) - def missing_meta( self ): - return self.datatype.missing_meta( self ) + def missing_meta( self, **kwd ): + return self.datatype.missing_meta( self, **kwd ) def as_display_type( self, type, **kwd ): return self.datatype.as_display_type( self, type, **kwd ) def display_peek( self ): @@ -258,8 +258,9 @@ class HistoryDatasetAssociation( object ): return self.datatype.find_conversion_destination( self, accepted_formats, datatypes_registry, **kwd ) def copy( self, copy_children = False, parent_id = None ): - des = HistoryDatasetAssociation( hid=self.hid, name=self.name, info=self.info, blurb=self.blurb, peek=self.peek, extension=self.extension, dbkey=self.dbkey, metadata=self._metadata, dataset = self.dataset, visible=self.visible, deleted=self.deleted, parent_id=parent_id, copied_from_history_dataset_association = self ) + des = HistoryDatasetAssociation( hid=self.hid, name=self.name, info=self.info, blurb=self.blurb, peek=self.peek, extension=self.extension, dbkey=self.dbkey, dataset = self.dataset, visible=self.visible, deleted=self.deleted, parent_id=parent_id, copied_from_history_dataset_association = self ) des.flush() + des.metadata = self.metadata #need to set after flushed, as MetadataFiles require dataset.id if copy_children: for child in self.children: child_copy = child.copy( copy_children = copy_children, parent_id = des.id ) @@ -564,6 +565,24 @@ class StoredWorkflowMenuEntry( object ): self.user = None self.order_index = None +class MetadataFile( object ): + def __init__( self, dataset = None, name = None ): + self.dataset = dataset + self.name = name + @property + def file_name( self ): + assert self.id is not None, "ID must be set before filename used (commit the object)" + path = os.path.join( Dataset.file_path, '_metadata_files', *directory_hash_id( self.id ) ) + # Create directory if it does not exist + try: + os.makedirs( path ) + except OSError, e: + # File Exists is okay, otherwise reraise + if e.errno != errno.EEXIST: + raise + # Return filename inside hashed directory + return os.path.abspath( os.path.join( path, "metadata_%d.dat" % self.id ) ) + ## ---- Utility methods ------------------------------------------------------- def directory_hash_id( id ): diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index c59e16968e0..7fc8aa957a4 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -238,6 +238,15 @@ StoredWorkflowMenuEntry.table = Table( "stored_workflow_menu_entry", metadata, Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "order_index", Integer ) ) +MetadataFile.table = Table( "metadata_file", metadata, + Column( "id", Integer, primary_key=True ), + Column( "name", String ), + Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, index=True, default=now, onupdate=now ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "purged", Boolean, index=True, default=False ) ) + # With the tables defined we can define the mappers and setup the # relationships between the model objects. @@ -363,6 +372,9 @@ assign_mapper( context, StoredWorkflowUserShareAssociation, StoredWorkflowUserSh assign_mapper( context, StoredWorkflowMenuEntry, StoredWorkflowMenuEntry.table, properties=dict( stored_workflow=relation( StoredWorkflow ) ) ) +assign_mapper( context, MetadataFile, MetadataFile.table, + properties=dict( dataset=relation( HistoryDatasetAssociation ) ) ) + def db_next_hid( self ): """ Override __next_hid to generate from the database in a concurrency diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index b3904436a63..25bdb29baed 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -196,7 +196,7 @@ class UploadToolAction( object ): temp = open( temp_name, "U" ) magic_check = temp.read( 2 ) temp.close() - if magic_check != datatypes.data.gzip_magic: + if magic_check != util.gzip_magic: return ( False, False ) CHUNK_SIZE = 2**15 # 32Kb gzipped_file = gzip.GzipFile( temp_name ) diff --git a/lib/galaxy/tools/parameters/__init__.py b/lib/galaxy/tools/parameters/__init__.py index 2aa3495ac71..f8cda91c0de 100644 --- a/lib/galaxy/tools/parameters/__init__.py +++ b/lib/galaxy/tools/parameters/__init__.py @@ -60,4 +60,4 @@ def params_from_strings( params, param_values, app, ignore_errors=False ): if key in params: value = params[key].value_from_basic( value, app, ignore_errors ) rval[ key ] = value - return rval \ No newline at end of file + return rval diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index 2e9ebd50fa8..2b1cbb5abb0 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -829,6 +829,8 @@ class DrillDownSelectToolParameter( ToolParameter ): options = [] for filter_key, filter_value in self.filtered.iteritems(): dataset = other_values[filter_key] + if dataset.__class__.__name__.endswith( "DatasetFilenameWrapper" ): #this is a bad way to check for this, but problems importing class ( due to circular imports? ) + dataset = dataset.dataset if dataset: for meta_key, meta_dict in filter_value.iteritems(): if dataset.metadata.spec[meta_key].param.to_string( dataset.metadata.get( meta_key ) ) == meta_dict['value']: diff --git a/lib/galaxy/tools/parameters/validation.py b/lib/galaxy/tools/parameters/validation.py index e2fb7e5c69a..443824e1dee 100644 --- a/lib/galaxy/tools/parameters/validation.py +++ b/lib/galaxy/tools/parameters/validation.py @@ -163,13 +163,15 @@ class MetadataValidator( Validator ): """ Validator that checks for missing metadata """ - def __init__( self, message=None ): + def __init__( self, message = None, check = "", skip = "" ): self.message = message + self.check = check.split( "," ) + self.skip = skip.split( "," ) @classmethod def from_element( cls, param, elem ): - return cls( elem.get( 'message', None ) ) + return cls( message=elem.get( 'message', None ), check=elem.get( 'check', "" ), skip=elem.get( 'skip', "" ) ) def validate( self, value, history=None ): - if value and value.missing_meta(): + if value and value.missing_meta( check = self.check, skip = self.skip ): if self.message is None: self.message = "Metadata missing, click the pencil icon in the history item to edit / save the metadata attributes" raise ValueError( self.message ) diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py index 6cfe70c2f84..097a5332f2f 100644 --- a/lib/galaxy/tools/util/maf_utilities.py +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -145,8 +145,16 @@ def maf_index_by_uid( maf_uid, index_location_file ): except: pass return None + +#return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created +def open_or_build_maf_index( maf_file, index_filename, species = None ): + try: + return ( bx.align.maf.Indexed( maf_file, index_filename = index_filename, keep_open = True, parse_e_rows = False ), None ) + except: + return build_maf_index( maf_file, species = species ) + -#builds and returns (index, index_filename) for specified maf_file +#builds and returns ( index, index_filename ) for specified maf_file def build_maf_index( maf_file, species = None ): indexes = bx.interval_index_file.Indexes() try: diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index f39219e3028..c15f8da3b5c 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -17,6 +17,8 @@ from elementtree import ElementTree log = logging.getLogger(__name__) _lock = threading.RLock() +gzip_magic = '\037\213' + def synchronized(func): """This wrapper will serialize access to 'func' to a single thread. Use it as a decorator.""" def caller(*params, **kparams): diff --git a/templates/dataset/edit_attributes.mako b/templates/dataset/edit_attributes.mako index ec54c03aaed..39936d24f4c 100644 --- a/templates/dataset/edit_attributes.mako +++ b/templates/dataset/edit_attributes.mako @@ -65,6 +65,9 @@ if they are not accurate. + %if data.missing_meta(): +
Required metadata values are missing. Some of these values may not be editable by the user. Selecting "Auto-detect" will attempt to fix these values.
+ %endif diff --git a/tools/data_source/data_source.py b/tools/data_source/data_source.py index acd6f77861d..2d48120aa33 100644 --- a/tools/data_source/data_source.py +++ b/tools/data_source/data_source.py @@ -2,7 +2,8 @@ #Retreives data from UCSC and stores in a file. UCSC parameters are provided in the input/output file. import urllib, sys, os, gzip, tempfile, shutil from galaxy import eggs -from galaxy.datatypes import data +#from galaxy.datatypes import data +from galaxy.util import gzip_magic assert sys.version_info[:2] >= ( 2, 4 ) @@ -14,7 +15,7 @@ def check_gzip( filename ): temp = open( filename, "U" ) magic_check = temp.read( 2 ) temp.close() - if magic_check != data.gzip_magic: + if magic_check != gzip_magic: return False return True diff --git a/tools/maf/genebed_maf_to_fasta.xml b/tools/maf/genebed_maf_to_fasta.xml index 22293b89ce9..e6897ec8ff5 100644 --- a/tools/maf/genebed_maf_to_fasta.xml +++ b/tools/maf/genebed_maf_to_fasta.xml @@ -1,6 +1,6 @@ given a set of coding exon intervals - #if $maf_source_type.maf_source == "user":#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_file --interval_file=$input1 --output_file=$out_file1 --mafSourceType=$maf_source_type.maf_source --geneBED --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} + #if $maf_source_type.maf_source == "user":#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_file --mafIndex=$maf_source_type.maf_file.metadata.maf_index --interval_file=$input1 --output_file=$out_file1 --mafSourceType=$maf_source_type.maf_source --geneBED --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} #else:#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_identifier --interval_file=$input1 --output_file=$out_file1 --mafSourceType=$maf_source_type.maf_source --geneBED --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} #end if diff --git a/tools/maf/interval2maf.py b/tools/maf/interval2maf.py index 498f51c8f23..99a5e66989b 100755 --- a/tools/maf/interval2maf.py +++ b/tools/maf/interval2maf.py @@ -16,6 +16,7 @@ usage: %prog maf_file [options] -S, --strandCol=S: Column of Strand -t, --mafType=t: Type of MAF source to use -m, --mafFile=m: Path of source MAF file, if not using cached version + -I, --mafIndex=I: Path of precomputed source MAF file index, if not using cached version -i, --interval_file=i: Input interval file -o, --output_file=o: Output MAF file -p, --species=p: Species to include in output @@ -92,7 +93,7 @@ def __main__(): print >> sys.stderr, "The MAF source specified (%s) appears to be invalid." % ( options.mafType ) sys.exit() elif options.mafFile: - index, index_filename = maf_utilities.build_maf_index( options.mafFile, species = [dbkey] ) + index, index_filename = maf_utilities.open_or_build_maf_index( options.mafFile, options.mafIndex, species = [dbkey] ) if index is None: print >> sys.stderr, "Your MAF file appears to be malformed." sys.exit() diff --git a/tools/maf/interval2maf.xml b/tools/maf/interval2maf.xml index 92816fc33cc..aceac0d36db 100644 --- a/tools/maf/interval2maf.xml +++ b/tools/maf/interval2maf.xml @@ -1,7 +1,7 @@ given a set of genomic intervals - #if $maf_source_type.maf_source == "user":#interval2maf.py --dbkey=${input1.dbkey} --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafFile=$maf_source_type.mafFile --interval_file=$input1 --output_file=$out_file1 --mafIndexFile=${GALAXY_DATA_INDEX_DIR}/maf_index.loc + #if $maf_source_type.maf_source == "user":#interval2maf.py --dbkey=${input1.dbkey} --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafFile=$maf_source_type.mafFile --mafIndex=$maf_source_type.mafFile.metadata.maf_index --interval_file=$input1 --output_file=$out_file1 --mafIndexFile=${GALAXY_DATA_INDEX_DIR}/maf_index.loc #else:#interval2maf.py --dbkey=${input1.dbkey} --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafType=$maf_source_type.mafType --interval_file=$input1 --output_file=$out_file1 --mafIndexFile=${GALAXY_DATA_INDEX_DIR}/maf_index.loc #end if diff --git a/tools/maf/interval_maf_to_merged_fasta.py b/tools/maf/interval_maf_to_merged_fasta.py index e8c5308dac2..3e74e916e16 100644 --- a/tools/maf/interval_maf_to_merged_fasta.py +++ b/tools/maf/interval_maf_to_merged_fasta.py @@ -15,6 +15,7 @@ usage: %prog maf_file [options] -G, --geneBED: Input is a Gene BED file, process and join exons as one region -t, --mafSourceType=t: Type of MAF source to use -m, --mafSource=m: Path of source MAF file, if not using cached version + -I, --mafIndex=I: Path of precomputed source MAF file index, if not using cached version -i, --interval_file=i: Input interval file -o, --output_file=o: Output MAF file -p, --species=p: Species to include in output @@ -105,7 +106,7 @@ def __main__(): stop_err( "The MAF source specified (%s) appears to be invalid." % ( options.mafSource ) ) elif options.mafSourceType.lower() in ["user"]: #index maf for use here, need to remove index_file when finished - index, index_filename = maf_utilities.build_maf_index( options.mafSource, species = [primary_species] ) + index, index_filename = maf_utilities.open_or_build_maf_index( options.mafSource, options.mafIndex, species = [primary_species] ) if index is None: stop_err( "Your MAF file appears to be malformed." ) else: diff --git a/tools/maf/interval_maf_to_merged_fasta.xml b/tools/maf/interval_maf_to_merged_fasta.xml index 98463920728..facfe79d458 100644 --- a/tools/maf/interval_maf_to_merged_fasta.xml +++ b/tools/maf/interval_maf_to_merged_fasta.xml @@ -1,6 +1,6 @@ given a set of genomic intervals - #if $maf_source_type.maf_source == "user":#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_file --interval_file=$input1 --output_file=$out_file1 --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafSourceType=$maf_source_type.maf_source --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} + #if $maf_source_type.maf_source == "user":#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_file --mafIndex=$maf_source_type.maf_file.metadata.maf_index --interval_file=$input1 --output_file=$out_file1 --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafSourceType=$maf_source_type.maf_source --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} #else:#interval_maf_to_merged_fasta.py --dbkey=$dbkey --species=$maf_source_type.species --mafSource=$maf_source_type.maf_identifier --interval_file=$input1 --output_file=$out_file1 --chromCol=${input1.metadata.chromCol} --startCol=${input1.metadata.startCol} --endCol=${input1.metadata.endCol} --strandCol=${input1.metadata.strandCol} --mafSourceType=$maf_source_type.maf_source --mafIndexFileDir=${GALAXY_DATA_INDEX_DIR} #end if diff --git a/tools/maf/maf_stats.py b/tools/maf/maf_stats.py index 98fa6a3f0d5..8f38e9bcbb7 100644 --- a/tools/maf/maf_stats.py +++ b/tools/maf/maf_stats.py @@ -31,10 +31,14 @@ def __main__(): else: summary = False mafIndexFile = "%s/maf_index.loc" % sys.argv[9] + try: + maf_index_filename = sys.argv[10].strip() + except: + maf_index_filename = None index = index_filename = None if maf_source_type == "user": #index maf for use here - index, index_filename = maf_utilities.build_maf_index( input_maf_filename, species = [dbkey] ) + index, index_filename = maf_utilities.open_or_build_maf_index( input_maf_filename, maf_index_filename, species = [dbkey] ) if index is None: print >>sys.stderr, "Your MAF file appears to be malformed." sys.exit() diff --git a/tools/maf/maf_stats.xml b/tools/maf/maf_stats.xml index a796f017077..fe89b43f291 100644 --- a/tools/maf/maf_stats.xml +++ b/tools/maf/maf_stats.xml @@ -7,7 +7,10 @@ #else: $maf_source_type.maf_source $maf_source_type.mafType $input1 $out_file1 $dbkey ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} $summary #end if - ${GALAXY_DATA_INDEX_DIR} + ${GALAXY_DATA_INDEX_DIR} + #if $maf_source_type.maf_source == "user": + $input2.metadata.maf_index + #end if diff --git a/tools/stats/filtering.py b/tools/stats/filtering.py index bb3d4d2c953..6826a9bed83 100644 --- a/tools/stats/filtering.py +++ b/tools/stats/filtering.py @@ -4,7 +4,6 @@ import sys, sets, re, os.path from galaxy import eggs -from galaxy.datatypes import metadata assert sys.version_info[:2] >= ( 2, 4 ) diff --git a/tools/visualization/GMAJ.xml b/tools/visualization/GMAJ.xml index cc40bd3ded7..26b4ad6a09b 100644 --- a/tools/visualization/GMAJ.xml +++ b/tools/visualization/GMAJ.xml @@ -2,7 +2,9 @@ Multiple Alignment Viewer GMAJ.py $out_file1 $maf_input $gmaj_file $filenames_file - + + + @@ -103,9 +105,19 @@ nowarn = $nowarn #set $seq_count = 0 #for $annotation_count, $annotation in $enumerate( $annotations ): -#if $annotation.annotation_style.style == "galaxy": -#if $maf_input.dataset.metadata.species_chromosomes and $annotation.annotation_style['species'].value in $maf_input.dataset.metadata.species_chromosomes and $maf_input.dataset.metadata.species_chromosomes[$annotation.annotation_style['species'].value]: -#set $seq_names = [ "%s.%s" % ( $annotation.annotation_style['species'].value, $chrom ) for $chrom in $maf_input.dataset.metadata.species_chromosomes[$annotation.annotation_style['species'].value]] +#if $annotation.annotation_style.style == "galaxy": +#set $species_chromosomes = {} +#if $maf_input.dataset.metadata.species_chromosomes: +#for $line in open( $maf_input.dataset.metadata.species_chromosomes.file_name ): +#set $fields = $line.split( "\t" ) +#if $fields: +#set $spec = $fields.pop( 0 ) +#set $species_chromosomes[spec] = $fields +#end if +#end for +#end if +#if $species_chromosomes and $annotation.annotation_style['species'].value in $species_chromosomes and $species_chromosomes[$annotation.annotation_style['species'].value]: +#set $seq_names = [ "%s.%s" % ( $annotation.annotation_style['species'].value, $chrom ) for $chrom in $species_chromosomes[$annotation.annotation_style['species'].value]] #else: #set $seq_names = [$annotation.annotation_style['species']] #end if @@ -171,4 +183,4 @@ For detailed information on GMAJ, click here_. Gmaj is a tool for viewing and manipulating Generalized Multiple Alignments (GMAs) produced by programs such as TBA (though it can also be used with maf-format alignments from other sources). It can display interactive graphical and text representations of the alignments, a diagram showing the locations of exons and repeats, and other annotations -- all with the user's choice of reference sequence. - \ No newline at end of file +