diff --git a/cron/updateucsc.sh.sample b/cron/updateucsc.sh.sample index fa33d659d34..ff4350bbf43 100644 --- a/cron/updateucsc.sh.sample +++ b/cron/updateucsc.sh.sample @@ -1,6 +1,6 @@ #!/bin/sh # -# Script to update UCSC static tables. The idea is to update, but if +# Script to update UCSC shared data tables. The idea is to update, but if # the update fails, not replace current data/tables with error # messages. @@ -9,49 +9,49 @@ GALAXY=/galaxy/path export PYTHONPATH=${GALAXY}/modules:${GALAXY}/eggs:${GALAXY}/lib # setup directories -mkdir ${GALAXY}/static/ucsc/new -mkdir ${GALAXY}/static/ucsc/chrom/new +mkdir ${GALAXY}/tool-data/shared/ucsc/new +mkdir ${GALAXY}/tool-data/shared/ucsc/chrom/new date -echo "Updating UCSC static tables." +echo "Updating UCSC shared data tables." # Try to build "builds.txt" echo "Updating builds.txt" -python ${GALAXY}/cron/parse_builds.py > ${GALAXY}/static/ucsc/new/builds.txt +python ${GALAXY}/cron/parse_builds.py > ${GALAXY}/tool-data/shared/ucsc/new/builds.txt if [ $? -eq 0 ] then - cp -uf ${GALAXY}/static/ucsc/new/builds.txt ${GALAXY}/static/ucsc/builds.txt + cp -uf ${GALAXY}/tool-data/shared/ucsc/new/builds.txt ${GALAXY}/tool-data/shared/ucsc/builds.txt else echo "Failed to update builds.txt" >&2 fi # Try to build ucsc_build_sites.txt echo "Updating ucsc_build_sites.txt" -python ${GALAXY}/cron/parse_builds_3_sites.py > ${GALAXY}/static/ucsc/new/ucsc_build_sites.txt +python ${GALAXY}/cron/parse_builds_3_sites.py > ${GALAXY}/tool-data/shared/ucsc/new/ucsc_build_sites.txt if [ $? -eq 0 ] then - cp -uf ${GALAXY}/static/ucsc/new/ucsc_build_sites.txt ${GALAXY}/static/ucsc/ucsc_build_sites.txt + cp -uf ${GALAXY}/tool-data/shared/ucsc/new/ucsc_build_sites.txt ${GALAXY}/tool-data/shared/ucsc/ucsc_build_sites.txt else echo "Failed to update builds.txt" >&2 fi # Try to build chromInfo tables echo "Building chromInfo tables." -python ${GALAXY}/cron/build_chrom_db.py ${GALAXY}/static/ucsc/chrom/new/ ${GALAXY}/static/ucsc/builds.txt +python ${GALAXY}/cron/build_chrom_db.py ${GALAXY}/tool-data/shared/ucsc/chrom/new/ ${GALAXY}/tool-data/shared/ucsc/builds.txt if [ $? -eq 0 ] then - cp -uf ${GALAXY}/static/ucsc/chrom/new/*.len ${GALAXY}/static/ucsc/chrom/ + cp -uf ${GALAXY}/tool-data/shared/ucsc/chrom/new/*.len ${GALAXY}/tool-data/shared/ucsc/chrom/ else echo "Failed to update chromInfo tables." >&2 fi -rm -rf ${GALAXY}/static/ucsc/new -rm -rf ${GALAXY}/static/ucsc/chrom/new +rm -rf ${GALAXY}/tool-data/shared/ucsc/new +rm -rf ${GALAXY}/tool-data/shared/ucsc/chrom/new echo "Update complete." #Perform Manual Additions here echo "Adding Manual Builds." -python ${GALAXY}/cron/add_manual_builds.py ${GALAXY}/static/ucsc/manual_builds.txt ${GALAXY}/static/ucsc/builds.txt ${GALAXY}/static/ucsc/chrom/ +python ${GALAXY}/cron/add_manual_builds.py ${GALAXY}/tool-data/shared/ucsc/manual_builds.txt ${GALAXY}/tool-data/shared/ucsc/builds.txt ${GALAXY}/tool-data/shared/ucsc/chrom/ if [ $? -eq 0 ] then echo "Manual addition was successful." diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index a4e03dd5e3b..81b51f4f1f4 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -242,7 +242,10 @@ def get_gbrowse_sites_by_build(build): def read_dbnames(filename): """ Read build names from file """ - db_names = [] + class DBNames( list ): + default_value = "?" + default_name = "unspecified (?)" + db_names = DBNames() try: ucsc_builds = {} man_builds = [] #assume these are integers @@ -276,15 +279,15 @@ def read_dbnames(filename): ucsc_builds[db_base].sort() ucsc_builds[db_base].reverse() ucsc_builds[db_base] = [(build, name) for build_rev, build, name in ucsc_builds[db_base]] - db_names = db_names + ucsc_builds[db_base] - if len(db_names)>1 and len(man_builds)>0: db_names.append(('?', '----- Additional Species Are Below -----')) + db_names = DBNames( db_names + ucsc_builds[db_base] ) + if len( db_names ) > 1 and len( man_builds ) > 0: db_names.append( ( db_names.default_value, '----- Additional Species Are Below -----' ) ) man_builds.sort() man_builds = [(build, name) for name, build in man_builds] - db_names = db_names + man_builds + db_names = DBNames( db_names + man_builds ) except Exception, e: print "ERROR: Unable to read builds file:", e if len(db_names)<1: - db_names = [('?', 'unspecified (?)')] + db_names = DBNames( [( db_names.default_value, db_names.default_name )] ) return db_names def read_build_sites(filename): @@ -306,9 +309,9 @@ def read_build_sites(filename): return build_sites galaxy_root_path = os.path.join(__path__[0], "..","..","..") -dbnames = read_dbnames(os.path.join(galaxy_root_path,"static","ucsc","builds.txt")) #this list is used in edit attributes and the upload tool -ucsc_build_sites = read_build_sites(os.path.join(galaxy_root_path,"static","ucsc","ucsc_build_sites.txt")) #this list is used in history.tmpl -gbrowse_build_sites = read_build_sites(os.path.join(galaxy_root_path,"static","gbrowse","gbrowse_build_sites.txt")) #this list is used in history.tmpl +dbnames = read_dbnames( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "builds.txt" ) ) #this list is used in edit attributes and the upload tool +ucsc_build_sites = read_build_sites( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "ucsc_build_sites.txt" ) ) #this list is used in history.tmpl +gbrowse_build_sites = read_build_sites( os.path.join( galaxy_root_path, "tool-data", "shared", "gbrowse", "gbrowse_build_sites.txt" ) ) #this list is used in history.tmpl if __name__ == '__main__': import doctest, sys diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index 6e27916293e..6c3911c7a3a 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -187,7 +187,7 @@ class UniverseWebTransaction( base.DefaultWebTransaction ): initialize a new history with the first dbkey in util.dbnames which is currently ? unspecified (?) """ - history.genome_build = util.dbnames[0][0] + history.genome_build = util.dbnames.default_value if history.user_id is None and self.user is not None: history.user_id = self.user.id if self.galaxy_session_is_valid(): diff --git a/static/gbrowse/gbrowse_build_sites.txt b/tool-data/shared/gbrowse/gbrowse_build_sites.txt similarity index 100% rename from static/gbrowse/gbrowse_build_sites.txt rename to tool-data/shared/gbrowse/gbrowse_build_sites.txt diff --git a/static/ucsc/builds.txt b/tool-data/shared/ucsc/builds.txt similarity index 100% rename from static/ucsc/builds.txt rename to tool-data/shared/ucsc/builds.txt diff --git a/static/ucsc/manual_builds.txt b/tool-data/shared/ucsc/manual_builds.txt similarity index 100% rename from static/ucsc/manual_builds.txt rename to tool-data/shared/ucsc/manual_builds.txt diff --git a/static/ucsc/ucsc_build_sites.txt b/tool-data/shared/ucsc/ucsc_build_sites.txt similarity index 100% rename from static/ucsc/ucsc_build_sites.txt rename to tool-data/shared/ucsc/ucsc_build_sites.txt diff --git a/tools/annotation_profiler/annotation_profiler.xml b/tools/annotation_profiler/annotation_profiler.xml index 95fb2294ca0..e297902ac63 100644 --- a/tools/annotation_profiler/annotation_profiler.xml +++ b/tools/annotation_profiler/annotation_profiler.xml @@ -1,6 +1,6 @@ for a set of genomic intervals - annotation_profiler_for_interval.py -i $input1 -c $input1_chromCol -s $input1_startCol -e $input1_endCol -o $out_file1 $keep_empty -p /depot/data2/galaxy/annotation_profiler/$dbkey $summary -b 3 -t $table_names + annotation_profiler_for_interval.py -i $input1 -c $input1_chromCol -s $input1_startCol -e $input1_endCol -o $out_file1 $keep_empty -p /depot/data2/galaxy/annotation_profiler/$dbkey $summary -l ${GALAXY_DATA_INDEX_DIR}/shared/ucsc/chrom/${dbkey}.len -b 3 -t $table_names diff --git a/tools/annotation_profiler/annotation_profiler_for_interval.py b/tools/annotation_profiler/annotation_profiler_for_interval.py index f11edb55c6f..d6f33289f9f 100644 --- a/tools/annotation_profiler/annotation_profiler_for_interval.py +++ b/tools/annotation_profiler/annotation_profiler_for_interval.py @@ -111,8 +111,9 @@ class CachedCoverageReader: yield tablename, coverage, regions, index class TableCoverageSummary: - def __init__( self, coverage_reader ): + def __init__( self, coverage_reader, chrom_lengths ): self.coverage_reader = coverage_reader + self.chrom_lengths = chrom_lengths self.chromosome_coverage = {} self.total_interval_size = 0 self.total_interval_count = 0 @@ -126,10 +127,7 @@ class TableCoverageSummary: self.total_interval_size += ( end - start ) self.total_interval_count += 1 if chrom not in self.chromosome_coverage: - #utilize lengths file here, if possible, if not use 250mb - #currently, no valid method to provide location of lengths file by framework: - #gops_complement has it hard coded as dbfile = fileinput.FileInput( "static/ucsc/chrom/"+db+".len" ) - self.chromosome_coverage[chrom] = bx.bitset.BitSet( 250000000 ) + self.chromosome_coverage[chrom] = bx.bitset.BitSet( self.chrom_lengths.get( chrom ) ) self.chromosome_coverage[chrom].set_range( start, end - start ) for table_name, coverage, regions in self.coverage_reader.iter_table_coverage_regions_by_region( chrom, start, end ): if table_name not in self.table_coverage: @@ -207,19 +205,34 @@ def profile_per_interval( interval_filename, chrom_col, start_col, end_col, out_ out.write( "%s\t%s\t%s\t%s\n" % ( "\t".join( region.fields ), table_name, coverage, region_count ) ) out.close() -def profile_summary( interval_filename, chrom_col, start_col, end_col, out_filename, keep_empty, coverage_reader ): +def profile_summary( interval_filename, chrom_col, start_col, end_col, out_filename, keep_empty, coverage_reader, chrom_lengths ): out = open( out_filename, 'wb' ) - table_coverage_summary = TableCoverageSummary( coverage_reader ) + table_coverage_summary = TableCoverageSummary( coverage_reader, chrom_lengths ) for region in bx.intervals.io.NiceReaderWrapper( open( interval_filename, 'rb' ), chrom_col = chrom_col, start_col = start_col, end_col = end_col, fix_strand = True, return_header = False, return_comments = False ): table_coverage_summary.add_region( region.chrom, region.start, region.end ) - out.write( "#tableName\ttableChromosomeCoverage\ttableChromosomeCount\ttableRegionCoverage\ttableRegionCount\tallIntervalCount\tallIntervalSize\tallCoverage\tallTableRegionsOverlaped\tallIntervalsOverlapingTable\tnrIntervalCount\tnrIntervalSize\tnrCoverage\tnrTableRegionsOverlaped\tnrIntervalsOverlapingTable\n" )#\tstatistic\n" ) + out.write( "#tableName\ttableChromosomeCoverage\ttableChromosomeCount\ttableRegionCoverage\ttableRegionCount\tallIntervalCount\tallIntervalSize\tallCoverage\tallTableRegionsOverlaped\tallIntervalsOverlapingTable\tnrIntervalCount\tnrIntervalSize\tnrCoverage\tnrTableRegionsOverlaped\tnrIntervalsOverlapingTable\n" ) for table_name, table_chromosome_size, table_chromosome_count, table_region_coverage, table_region_count, total_interval_count, total_interval_size, total_coverage, table_regions_overlaped_count, interval_region_overlap_count, nr_interval_count, nr_region_size, nr_coverage, nr_table_regions_overlaped_count, nr_interval_table_overlap_count in table_coverage_summary.iter_table_coverage(): if keep_empty or total_coverage: #only output tables that have atleast 1 base covered unless empty are requested out.write( "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % ( table_name, table_chromosome_size, table_chromosome_count, table_region_coverage, table_region_count, total_interval_count, total_interval_size, total_coverage, table_regions_overlaped_count, interval_region_overlap_count, nr_interval_count, nr_region_size, nr_coverage, nr_table_regions_overlaped_count, nr_interval_table_overlap_count ) ) out.close() +class ChromosomeLengths: + def __init__( self, filename ): + self.chroms = {} + try: + for line in open( filename ): + try: + fields = line.strip().split( "\t" ) + self.chroms[fields[0]] = int( fields[1] ) + except: + continue + except: + pass + def get( self, name ): + return self.chroms.get( name, bx.bitset.MAX ) + def __main__(): parser = optparse.OptionParser() parser.add_option( @@ -259,6 +272,12 @@ def __main__(): type='str',default='/depot/data2/galaxy/annotation_profiler/hg18', help='Path to profiled data for this organism' ) + parser.add_option( + '-l','--lengths', + dest='lengths', + type='str',default='test-data/shared/ucsc/hg18.len', + help='Path to chromosome lengths data for this organism' + ) parser.add_option( '-t','--table_names', dest='table_names', @@ -292,7 +311,7 @@ def __main__(): coverage_reader = CachedCoverageReader( options.path, buffer = options.buffer, table_names = table_names ) if options.summary: - profile_summary( options.interval_filename, options.chrom_col - 1, options.start_col - 1, options.end_col -1, options.out_filename, options.keep_empty, coverage_reader ) + profile_summary( options.interval_filename, options.chrom_col - 1, options.start_col - 1, options.end_col -1, options.out_filename, options.keep_empty, coverage_reader, ChromosomeLengths( options.lengths ) ) else: profile_per_interval( options.interval_filename, options.chrom_col - 1, options.start_col - 1, options.end_col -1, options.out_filename, options.keep_empty, coverage_reader ) diff --git a/tools/filters/axt_to_lav.xml b/tools/filters/axt_to_lav.xml index a693285b5b6..39c3b734240 100644 --- a/tools/filters/axt_to_lav.xml +++ b/tools/filters/axt_to_lav.xml @@ -1,6 +1,6 @@ Converts an AXT formated file to LAV format - axt_to_lav.py ${GALAXY_DATA_INDEX_DIR}/$dbkey_1/seq/%s.nib:$dbkey_1:./static/ucsc/chrom/${dbkey_1}.len ${GALAXY_DATA_INDEX_DIR}/$dbkey_2/seq/%s.nib:$dbkey_2:./static/ucsc/chrom/${dbkey_2}.len $align_input $lav_file $seq_file1 $seq_file2 + axt_to_lav.py /depot/data2/galaxy/$dbkey_1/seq/%s.nib:$dbkey_1:${GALAXY_DATA_INDEX_DIR}/shared/ucsc/chrom/${dbkey_1}.len /depot/data2/galaxy/$dbkey_2/seq/%s.nib:$dbkey_2:${GALAXY_DATA_INDEX_DIR}/shared/ucsc/chrom/${dbkey_2}.len $align_input $lav_file $seq_file1 $seq_file2 diff --git a/tools/new_operations/complement.xml b/tools/new_operations/complement.xml index 6a8b7d70715..cccde19e0f6 100644 --- a/tools/new_operations/complement.xml +++ b/tools/new_operations/complement.xml @@ -1,6 +1,6 @@ intervals of a query - gops_complement.py $input1 $output -1 $input1_chromCol,$input1_startCol,$input1_endCol,$input1_strandCol -d $dbkey $allchroms + gops_complement.py $input1 $output -1 $input1_chromCol,$input1_startCol,$input1_endCol,$input1_strandCol -l ${GALAXY_DATA_INDEX_DIR}/shared/ucsc/chrom/${dbkey}.len $allchroms diff --git a/tools/new_operations/gops_complement.py b/tools/new_operations/gops_complement.py index 09a78e25e51..7615f83f4e2 100644 --- a/tools/new_operations/gops_complement.py +++ b/tools/new_operations/gops_complement.py @@ -4,7 +4,7 @@ Complement regions. usage: %prog in_file out_file -1, --cols1=N,N,N,N: Columns for chrom, start, end, strand in file - -d, --db=N: Database name (for determining chromosome lengths) + -l, --lengths=N: Filename of .len file for species (chromosome lengths) -a, --all: Complement all chromosomes (Genome-wide complement) """ from galaxy import eggs @@ -29,7 +29,7 @@ def main(): options, args = doc_optparse.parse( __doc__ ) try: chr_col_1, start_col_1, end_col_1, strand_col_1 = parse_cols_arg( options.cols1 ) - db = options.db + lengths = options.lengths if options.all: allchroms = True in_fname, out_fname = args except: @@ -46,7 +46,7 @@ def main(): chroms = list() # dbfile is used to determine the length of each chromosome. The lengths # are added to the lens dict and passed copmlement operation code in bx. - dbfile = fileinput.FileInput( "static/ucsc/chrom/"+db+".len" ) + dbfile = fileinput.FileInput( lengths ) if dbfile: if not allchroms: