diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 03d56cac84b..f704bd72bb0 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -11,6 +11,7 @@ import galaxy.queues import galaxy.quota import galaxy.security from galaxy import config, job_metrics, jobs +from galaxy.config_watchers import ConfigWatchers from galaxy.containers import build_container_interfaces from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.folders import FolderManager @@ -40,7 +41,6 @@ from galaxy.visualization.plugins.registry import VisualizationsRegistry from galaxy.web import url_for from galaxy.web.proxy import ProxyManager from galaxy.web_stack import application_stack_instance -from galaxy.webapps.galaxy.config_watchers import ConfigWatchers from galaxy.webhooks import WebhooksRegistry from tool_shed.galaxy_install import ( installed_repository_manager, diff --git a/lib/galaxy/webapps/galaxy/config_watchers.py b/lib/galaxy/config_watchers.py similarity index 100% rename from lib/galaxy/webapps/galaxy/config_watchers.py rename to lib/galaxy/config_watchers.py diff --git a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py index 98e6af16c01..1b893f1ab5d 100644 --- a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.py @@ -6,7 +6,7 @@ import sys import bx.align.maf -from galaxy.tools.util import maf_utilities +from galaxy.datatypes.util import maf_utilities assert sys.version_info[:2] >= (2, 6) diff --git a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py index 51049ed54c6..a19ab25cabb 100644 --- a/lib/galaxy/datatypes/converters/maf_to_interval_converter.py +++ b/lib/galaxy/datatypes/converters/maf_to_interval_converter.py @@ -6,7 +6,7 @@ import sys import bx.align.maf -from galaxy.tools.util import maf_utilities +from galaxy.datatypes.util import maf_utilities assert sys.version_info[:2] >= (2, 6) diff --git a/lib/galaxy/datatypes/test/temp2.txt b/lib/galaxy/datatypes/test/temp2.txt new file mode 100644 index 00000000000..2c3f803432c --- /dev/null +++ b/lib/galaxy/datatypes/test/temp2.txt @@ -0,0 +1,2 @@ +1 2 +3 4 diff --git a/lib/galaxy/datatypes/util/maf_utilities.py b/lib/galaxy/datatypes/util/maf_utilities.py new file mode 100644 index 00000000000..3a5aa531bb2 --- /dev/null +++ b/lib/galaxy/datatypes/util/maf_utilities.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python +""" +Provides wrappers and utilities for working with MAF files and alignments. +""" +# Dan Blankenberg +from __future__ import print_function + +import functools +import logging +import os +import resource +import sys +import tempfile +from copy import deepcopy +from errno import EMFILE + +import bx.align.maf +import bx.interval_index_file +import bx.intervals +from six.moves import xrange + +try: + from string import maketrans +except ImportError: + maketrans = str.maketrans + +assert sys.version_info[:2] >= (2, 6) + +log = logging.getLogger(__name__) + +GAP_CHARS = ['-'] +SRC_SPLIT_CHAR = '.' + + +def src_split(src): + fields = src.split(SRC_SPLIT_CHAR, 1) + spec = fields.pop(0) + if fields: + chrom = fields.pop(0) + else: + chrom = spec + return spec, chrom + + +def src_merge(spec, chrom, contig=None): + if None in [spec, chrom]: + spec = chrom = spec or chrom + return bx.align.maf.src_merge(spec, chrom, contig) + + +def get_species_in_block(block): + species = [] + for c in block.components: + spec, chrom = src_split(c.src) + if spec not in species: + species.append(spec) + return species + + +def tool_fail(msg="Unknown Error"): + print("Fatal Error: %s" % msg, file=sys.stderr) + sys.exit() + + +class TempFileHandler(object): + ''' + Handles creating, opening, closing, and deleting of Temp files, with a + maximum number of files open at one time. + ''' + + DEFAULT_MAX_OPEN_FILES = max(resource.getrlimit(resource.RLIMIT_NOFILE)[0] / 2, 1) + + def __init__(self, max_open_files=None, **kwds): + if max_open_files is None: + max_open_files = self.DEFAULT_MAX_OPEN_FILES + self.max_open_files = max_open_files + self.files = [] + self.open_file_indexes = [] + self.kwds = kwds + + def get_open_tempfile(self, index=None, **kwds): + if index is not None and index in self.open_file_indexes: + self.open_file_indexes.remove(index) + else: + if self.max_open_files: + while len(self.open_file_indexes) >= self.max_open_files: + self.close(self.open_file_indexes[0]) + if index is None: + index = len(self.files) + temp_kwds = dict(self.kwds) + temp_kwds.update(kwds) + # Being able to use delete=True here, would simplify a bit, + # but we support python2.4 in these tools + while True: + try: + tmp_file = tempfile.NamedTemporaryFile(**temp_kwds) + filename = tmp_file.name + break + except OSError as e: + if self.open_file_indexes and e.errno == EMFILE: + self.max_open_files = len(self.open_file_indexes) + self.close(self.open_file_indexes[0]) + else: + raise e + tmp_file.close() + self.files.append(open(filename, 'w+b')) + else: + while True: + try: + self.files[index] = open(self.files[index].name, 'r+b') + break + except OSError as e: + if self.open_file_indexes and e.errno == EMFILE: + self.max_open_files = len(self.open_file_indexes) + self.close(self.open_file_indexes[0]) + else: + raise e + self.files[index].seek(0, 2) + self.open_file_indexes.append(index) + return index, self.files[index] + + def close(self, index, delete=False): + if index in self.open_file_indexes: + self.open_file_indexes.remove(index) + rval = self.files[index].close() + if delete: + try: + os.unlink(self.files[index].name) + except OSError: + pass + return rval + + def flush(self, index): + if index in self.open_file_indexes: + self.files[index].flush() + + def __del__(self): + for i in xrange(len(self.files)): + self.close(i, delete=True) + + +# an object corresponding to a reference layered alignment +class RegionAlignment(object): + + DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca") + MAX_SEQUENCE_SIZE = sys.maxsize # Maximum length of sequence allowed + + def __init__(self, size, species=[], temp_file_handler=None): + assert size <= self.MAX_SEQUENCE_SIZE, "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % (size, self.MAX_SEQUENCE_SIZE) + self.size = size + if not temp_file_handler: + temp_file_handler = TempFileHandler() + self.temp_file_handler = temp_file_handler + self.sequences = {} + if not isinstance(species, list): + species = [species] + for spec in species: + self.add_species(spec) + + # add a species to the alignment + def add_species(self, species): + # make temporary sequence files + file_index, fh = self.temp_file_handler.get_open_tempfile() + self.sequences[species] = file_index + fh.write("-" * self.size) + + # returns the names for species found in alignment, skipping names as requested + def get_species_names(self, skip=[]): + if not isinstance(skip, list): + skip = [skip] + names = list(self.sequences.keys()) + for name in skip: + try: + names.remove(name) + except ValueError: + pass + return names + + # returns the sequence for a species + def get_sequence(self, species): + file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species]) + fh.seek(0) + return fh.read() + + # returns the reverse complement of the sequence for a species + def get_sequence_reverse_complement(self, species): + complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)] + complement.reverse() + return "".join(complement) + + # sets a position for a species + def set_position(self, index, species, base): + if len(base) != 1: + raise Exception("A genomic position can only have a length of 1.") + return self.set_range(index, species, base) + # sets a range for a species + + def set_range(self, index, species, bases): + if index >= self.size or index < 0: + raise Exception("Your index (%i) is out of range (0 - %i)." % (index, self.size - 1)) + if len(bases) == 0: + raise Exception("A set of genomic positions can only have a positive length.") + if species not in self.sequences.keys(): + self.add_species(species) + file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species]) + fh.seek(index) + fh.write(bases) + + # Flush temp file of specified species, or all species + def flush(self, species=None): + if species is None: + species = self.sequences.keys() + elif not isinstance(species, list): + species = [species] + for spec in species: + self.temp_file_handler.flush(self.sequences[spec]) + + +class GenomicRegionAlignment(RegionAlignment): + + def __init__(self, start, end, species=[], temp_file_handler=None): + RegionAlignment.__init__(self, end - start, species, temp_file_handler=temp_file_handler) + self.start = start + self.end = end + + +class SplicedAlignment(object): + + DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca") + + def __init__(self, exon_starts, exon_ends, species=[], temp_file_handler=None): + if not isinstance(exon_starts, list): + exon_starts = [exon_starts] + if not isinstance(exon_ends, list): + exon_ends = [exon_ends] + assert len(exon_starts) == len(exon_ends), "The number of starts does not match the number of sizes." + self.exons = [] + if not temp_file_handler: + temp_file_handler = TempFileHandler() + self.temp_file_handler = temp_file_handler + for i in range(len(exon_starts)): + self.exons.append(GenomicRegionAlignment(exon_starts[i], exon_ends[i], species, temp_file_handler=temp_file_handler)) + + # returns the names for species found in alignment, skipping names as requested + def get_species_names(self, skip=[]): + if not isinstance(skip, list): + skip = [skip] + names = [] + for exon in self.exons: + for name in exon.get_species_names(skip=skip): + if name not in names: + names.append(name) + return names + + # returns the sequence for a species + def get_sequence(self, species): + index, fh = self.temp_file_handler.get_open_tempfile() + for exon in self.exons: + if species in exon.get_species_names(): + seq = exon.get_sequence(species) + # we need to refetch fh here, since exon.get_sequence( species ) uses a tempfile + # and if max==1, it would close fh + index, fh = self.temp_file_handler.get_open_tempfile(index) + fh.write(seq) + else: + fh.write("-" * exon.size) + fh.seek(0) + rval = fh.read() + self.temp_file_handler.close(index, delete=True) + return rval + + # returns the reverse complement of the sequence for a species + def get_sequence_reverse_complement(self, species): + complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)] + complement.reverse() + return "".join(complement) + + # Start and end of coding region + @property + def start(self): + return self.exons[0].start + + @property + def end(self): + return self.exons[-1].end + + +# Open a MAF index using a UID +def maf_index_by_uid(maf_uid, index_location_file): + for line in open(index_location_file): + try: + # read each line, if not enough fields, go to next line + if line[0:1] == "#": + continue + fields = line.split('\t') + if maf_uid == fields[1]: + try: + maf_files = fields[4].replace("\n", "").replace("\r", "").split(",") + return bx.align.maf.MultiIndexed(maf_files, keep_open=True, parse_e_rows=False) + except Exception as e: + raise Exception('MAF UID (%s) found, but configuration appears to be malformed: %s' % (maf_uid, e)) + except Exception: + pass + return None + + +# return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created +def open_or_build_maf_index(maf_file, index_filename, species=None): + try: + return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), None) + except Exception: + return build_maf_index(maf_file, species=species) + + +def build_maf_index_species_chromosomes(filename, index_species=None): + species = [] + species_chromosomes = {} + indexes = bx.interval_index_file.Indexes() + blocks = 0 + try: + maf_reader = bx.align.maf.Reader(open(filename)) + while True: + pos = maf_reader.file.tell() + block = next(maf_reader) + if block is None: + break + blocks += 1 + for c in block.components: + spec = c.src + chrom = None + if "." in spec: + spec, chrom = spec.split(".", 1) + if spec not in species: + species.append(spec) + species_chromosomes[spec] = [] + if chrom and chrom not in species_chromosomes[spec]: + species_chromosomes[spec].append(chrom) + if index_species is None or spec in index_species: + forward_strand_start = c.forward_strand_start + forward_strand_end = c.forward_strand_end + try: + forward_strand_start = int(forward_strand_start) + forward_strand_end = int(forward_strand_end) + except ValueError: + continue # start and end are not integers, can't add component to index, goto next component + # this likely only occurs when parse_e_rows is True? + # could a species exist as only e rows? should the + if forward_strand_end > forward_strand_start: + # require positive length; i.e. certain lines have start = end = 0 and cannot be indexed + indexes.add(c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size) + except Exception as e: + # most likely a bad MAF + log.debug('Building MAF index on %s failed: %s' % (filename, e)) + return (None, [], {}, 0) + return (indexes, species, species_chromosomes, blocks) + + +# builds and returns ( index, index_filename ) for specified maf_file +def build_maf_index(maf_file, species=None): + indexes, found_species, species_chromosomes, blocks = build_maf_index_species_chromosomes(maf_file, species) + if indexes is not None: + fd, index_filename = tempfile.mkstemp() + out = os.fdopen(fd, 'w') + indexes.write(out) + out.close() + return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), index_filename) + return (None, None) + + +def component_overlaps_region(c, region): + if c is None: + return False + start, end = c.get_forward_strand_start(), c.get_forward_strand_end() + if region.start >= end or region.end <= start: + return False + return True + + +def chop_block_by_region(block, src, region, species=None, mincols=0): + # This chopping method was designed to maintain consistency with how start/end padding gaps have been working in Galaxy thus far: + # behavior as seen when forcing blocks to be '+' relative to src sequence (ref) and using block.slice_by_component( ref, slice_start, slice_end ) + # whether-or-not this is the 'correct' behavior is questionable, but this will at least maintain consistency + # comments welcome + slice_start = block.text_size # max for the min() + slice_end = 0 # min for the max() + old_score = block.score # save old score for later use + # We no longer assume only one occurance of src per block, so we need to check them all + for c in iter_components_by_src(block, src): + if component_overlaps_region(c, region): + if c.text is not None: + rev_strand = False + if c.strand == "-": + # We want our coord_to_col coordinates to be returned from positive stranded component + rev_strand = True + c = c.reverse_complement() + start = max(region.start, c.start) + end = min(region.end, c.end) + start = c.coord_to_col(start) + end = c.coord_to_col(end) + if rev_strand: + # need to orient slice coordinates to the original block direction + slice_len = end - start + end = len(c.text) - start + start = end - slice_len + slice_start = min(start, slice_start) + slice_end = max(end, slice_end) + + if slice_start < slice_end: + block = block.slice(slice_start, slice_end) + if block.text_size > mincols: + # restore old score, may not be accurate, but it is better than 0 for everything? + block.score = old_score + if species is not None: + block = block.limit_to_species(species) + block.remove_all_gap_columns() + return block + return None + + +def orient_block_by_region(block, src, region, force_strand=None): + # loop through components matching src, + # make sure each of these components overlap region + # cache strand for each of overlaping regions + # if force_strand / region.strand not in strand cache, reverse complement + # we could have 2 sequences with same src, overlapping region, on different strands, this would cause no reverse_complementing + strands = [c.strand for c in iter_components_by_src(block, src) if component_overlaps_region(c, region)] + if strands and (force_strand is None and region.strand not in strands) or (force_strand is not None and force_strand not in strands): + block = block.reverse_complement() + return block + + +def get_oriented_chopped_blocks_for_region(index, src, region, species=None, mincols=0, force_strand=None): + for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand): + yield block + + +def get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0, force_strand=None): + for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): + yield orient_block_by_region(block, src, region, force_strand), idx, offset + + +# split a block with multiple occurances of src into one block per src +def iter_blocks_split_by_src(block, src): + for src_c in iter_components_by_src(block, src): + new_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) + new_block.text_size = block.text_size + for c in block.components: + if c == src_c or c.src != src: + new_block.add_component(deepcopy(c)) # components have reference to alignment, don't want to lose reference to original alignment block in original components + yield new_block + + +# split a block into multiple blocks with all combinations of a species appearing only once per block +def iter_blocks_split_by_species(block, species=None): + def __split_components_by_species(components_by_species, new_block): + if components_by_species: + # more species with components to add to this block + components_by_species = deepcopy(components_by_species) + spec_comps = components_by_species.pop(0) + for c in spec_comps: + newer_block = deepcopy(new_block) + newer_block.add_component(deepcopy(c)) + for value in __split_components_by_species(components_by_species, newer_block): + yield value + else: + # no more components to add, yield this block + yield new_block + + # divide components by species + spec_dict = {} + if not species: + species = [] + for c in block.components: + spec, chrom = src_split(c.src) + if spec not in spec_dict: + spec_dict[spec] = [] + species.append(spec) + spec_dict[spec].append(c) + else: + for spec in species: + spec_dict[spec] = [] + for c in iter_components_by_src_start(block, spec): + spec_dict[spec].append(c) + + empty_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) # should we copy attributes? + empty_block.text_size = block.text_size + # call recursive function to split into each combo of spec/blocks + for value in __split_components_by_species(list(spec_dict.values()), empty_block): + sort_block_components_by_block(value, block) # restore original component order + yield value + + +# generator yielding only chopped and valid blocks for a specified region +def get_chopped_blocks_for_region(index, src, region, species=None, mincols=0): + for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): + yield block + + +def get_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0): + for block, idx, offset in index.get_as_iterator_with_index_and_offset(src, region.start, region.end): + block = chop_block_by_region(block, src, region, species, mincols) + if block is not None: + yield block, idx, offset + + +# returns a filled region alignment for specified regions +def get_region_alignment(index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None): + if species is not None: + alignment = RegionAlignment(end - start, species, temp_file_handler=temp_file_handler) + else: + alignment = RegionAlignment(end - start, primary_species, temp_file_handler=temp_file_handler) + return fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps) + + +# reduces a block to only positions exisiting in the src provided +def reduce_block_by_primary_genome(block, species, chromosome, region_start): + # returns ( startIndex, {species:texts} + # where texts' contents are reduced to only positions existing in the primary genome + src = "%s.%s" % (species, chromosome) + ref = block.get_component_by_src(src) + start_offset = ref.start - region_start + species_texts = {} + for c in block.components: + species_texts[c.src.split('.')[0]] = list(c.text) + # remove locations which are gaps in the primary species, starting from the downstream end + for i in range(len(species_texts[species]) - 1, -1, -1): + if species_texts[species][i] == '-': + for text in species_texts.values(): + text.pop(i) + for spec, text in species_texts.items(): + species_texts[spec] = ''.join(text) + return (start_offset, species_texts) + + +# fills a region alignment +def fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True): + region = bx.intervals.Interval(start, end) + region.chrom = chrom + region.strand = strand + primary_src = "%s.%s" % (primary_species, chrom) + + # Order blocks overlaping this position by score, lowest first + blocks = [] + for block, idx, offset in index.get_as_iterator_with_index_and_offset(primary_src, start, end): + score = float(block.score) + for i in range(0, len(blocks)): + if score < blocks[i][0]: + blocks.insert(i, (score, idx, offset)) + break + else: + blocks.append((score, idx, offset)) + + # gap_chars_tuple = tuple( GAP_CHARS ) + gap_chars_str = ''.join(GAP_CHARS) + # Loop through ordered blocks and layer by increasing score + for block_dict in blocks: + for block in iter_blocks_split_by_species(block_dict[1].get_at_offset(block_dict[2])): # need to handle each occurance of sequence in block seperately + if component_overlaps_region(block.get_component_by_src(primary_src), region): + block = chop_block_by_region(block, primary_src, region, species, mincols) # chop block + block = orient_block_by_region(block, primary_src, region) # orient block + start_offset, species_texts = reduce_block_by_primary_genome(block, primary_species, chrom, start) + for spec, text in species_texts.items(): + # we should trim gaps from both sides, since these are not positions in this species genome (sequence) + text = text.rstrip(gap_chars_str) + gap_offset = 0 + # while text.startswith( gap_chars_tuple ): + while True in [text.startswith(gap_char) for gap_char in GAP_CHARS]: # python2.4 doesn't accept a tuple for .startswith() + gap_offset += 1 + text = text[1:] + if not text: + break + if text: + if overwrite_with_gaps: + alignment.set_range(start_offset + gap_offset, spec, text) + else: + for i, char in enumerate(text): + if char not in GAP_CHARS: + alignment.set_position(start_offset + gap_offset + i, spec, char) + return alignment + + +# returns a filled spliced region alignment for specified region with start and end lists +def get_spliced_region_alignment(index, primary_species, chrom, starts, ends, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None): + # create spliced alignment object + if species is not None: + alignment = SplicedAlignment(starts, ends, species, temp_file_handler=temp_file_handler) + else: + alignment = SplicedAlignment(starts, ends, [primary_species], temp_file_handler=temp_file_handler) + for exon in alignment.exons: + fill_region_alignment(exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps) + return alignment + + +# loop through string array, only return non-commented lines +def line_enumerator(lines, comment_start='#'): + i = 0 + for line in lines: + if not line.startswith(comment_start): + i += 1 + yield (i, line) + + +# read a GeneBed file, return list of starts, ends, raw fields +def get_starts_ends_fields_from_gene_bed(line): + # Starts and ends for exons + starts = [] + ends = [] + + fields = line.split() + # Requires atleast 12 BED columns + if len(fields) < 12: + raise Exception("Not a proper 12 column BED line (%s)." % line) + tx_start = int(fields[1]) + strand = fields[5] + if strand != '-': + strand = '+' # Default strand is + + cds_start = int(fields[6]) + cds_end = int(fields[7]) + + # Calculate and store starts and ends of coding exons + region_start, region_end = cds_start, cds_end + exon_starts = list(map(int, fields[11].rstrip(',\n').split(','))) + exon_starts = [x + tx_start for x in exon_starts] + exon_ends = list(map(int, fields[10].rstrip(',').split(','))) + exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)] + for start, end in zip(exon_starts, exon_ends): + start = max(start, region_start) + end = min(end, region_end) + if start < end: + starts.append(start) + ends.append(end) + return (starts, ends, fields) + + +def iter_components_by_src(block, src): + for c in block.components: + if c.src == src: + yield c + + +def get_components_by_src(block, src): + return [value for value in iter_components_by_src(block, src)] + + +def iter_components_by_src_start(block, src): + for c in block.components: + if c.src.startswith(src): + yield c + + +def get_components_by_src_start(block, src): + return [value for value in iter_components_by_src_start(block, src)] + + +def sort_block_components_by_block(block1, block2): + # orders the components in block1 by the index of the component in block2 + # block1 must be a subset of block2 + # occurs in-place + return block1.components.sort(key=functools.cmp_to_key(lambda x, y: block2.components.index(x) - block2.components.index(y))) + + +def get_species_in_maf(maf_filename): + species = [] + for block in bx.align.maf.Reader(open(maf_filename)): + for spec in get_species_in_block(block): + if spec not in species: + species.append(spec) + return species + + +def parse_species_option(species): + if species: + species = species.split(',') + if 'None' not in species: + return species + return None # provided species was '', None, or had 'None' in it + + +def remove_temp_index_file(index_filename): + try: + os.unlink(index_filename) + except Exception: + pass + +# Below are methods to deal with FASTA files + + +def get_fasta_header(component, attributes={}, suffix=None): + header = ">%s(%s):%i-%i|" % (component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end()) + for key, value in attributes.items(): + header = "%s%s=%s|" % (header, key, value) + if suffix: + header = "%s%s" % (header, suffix) + else: + header = "%s%s" % (header, src_split(component.src)[0]) + return header + + +def get_attributes_from_fasta_header(header): + if not header: + return {} + attributes = {} + header = header.lstrip('>') + header = header.strip() + fields = header.split('|') + try: + region = fields[0] + region = region.split('(', 1) + temp = region[0].split('.', 1) + attributes['species'] = temp[0] + if len(temp) == 2: + attributes['chrom'] = temp[1] + else: + attributes['chrom'] = temp[0] + region = region[1].split(')', 1) + attributes['strand'] = region[0] + region = region[1].lstrip(':').split('-') + attributes['start'] = int(region[0]) + attributes['end'] = int(region[1]) + except Exception: + # fields 0 is not a region coordinate + pass + if len(fields) > 2: + for i in range(1, len(fields) - 1): + prop = fields[i].split('=', 1) + if len(prop) == 2: + attributes[prop[0]] = prop[1] + if len(fields) > 1: + attributes['__suffix__'] = fields[-1] + return attributes + + +def iter_fasta_alignment(filename): + class fastaComponent(object): + def __init__(self, species, text=""): + self.species = species + self.text = text + + def extend(self, text): + self.text = self.text + text.replace('\n', '').replace('\r', '').strip() + # yields a list of fastaComponents for a FASTA file + f = open(filename, 'rb') + components = [] + # cur_component = None + while True: + line = f.readline() + if not line: + if components: + yield components + return + line = line.strip() + if not line: + if components: + yield components + components = [] + elif line.startswith('>'): + attributes = get_attributes_from_fasta_header(line) + components.append(fastaComponent(attributes['species'])) + elif components: + components[-1].extend(line) diff --git a/lib/galaxy/model/tool_shed_install/__init__.py b/lib/galaxy/model/tool_shed_install/__init__.py index 430642f60dc..1a7d611967f 100644 --- a/lib/galaxy/model/tool_shed_install/__init__.py +++ b/lib/galaxy/model/tool_shed_install/__init__.py @@ -4,7 +4,7 @@ import os from galaxy.util import asbool from galaxy.util.bunch import Bunch from galaxy.util.dictifiable import Dictifiable -from tool_shed.util import common_util +from galaxy.util.tool_shed import common_util log = logging.getLogger(__name__) diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py deleted file mode 100644 index 3a5aa531bb2..00000000000 --- a/lib/galaxy/tools/util/maf_utilities.py +++ /dev/null @@ -1,760 +0,0 @@ -#!/usr/bin/env python -""" -Provides wrappers and utilities for working with MAF files and alignments. -""" -# Dan Blankenberg -from __future__ import print_function - -import functools -import logging -import os -import resource -import sys -import tempfile -from copy import deepcopy -from errno import EMFILE - -import bx.align.maf -import bx.interval_index_file -import bx.intervals -from six.moves import xrange - -try: - from string import maketrans -except ImportError: - maketrans = str.maketrans - -assert sys.version_info[:2] >= (2, 6) - -log = logging.getLogger(__name__) - -GAP_CHARS = ['-'] -SRC_SPLIT_CHAR = '.' - - -def src_split(src): - fields = src.split(SRC_SPLIT_CHAR, 1) - spec = fields.pop(0) - if fields: - chrom = fields.pop(0) - else: - chrom = spec - return spec, chrom - - -def src_merge(spec, chrom, contig=None): - if None in [spec, chrom]: - spec = chrom = spec or chrom - return bx.align.maf.src_merge(spec, chrom, contig) - - -def get_species_in_block(block): - species = [] - for c in block.components: - spec, chrom = src_split(c.src) - if spec not in species: - species.append(spec) - return species - - -def tool_fail(msg="Unknown Error"): - print("Fatal Error: %s" % msg, file=sys.stderr) - sys.exit() - - -class TempFileHandler(object): - ''' - Handles creating, opening, closing, and deleting of Temp files, with a - maximum number of files open at one time. - ''' - - DEFAULT_MAX_OPEN_FILES = max(resource.getrlimit(resource.RLIMIT_NOFILE)[0] / 2, 1) - - def __init__(self, max_open_files=None, **kwds): - if max_open_files is None: - max_open_files = self.DEFAULT_MAX_OPEN_FILES - self.max_open_files = max_open_files - self.files = [] - self.open_file_indexes = [] - self.kwds = kwds - - def get_open_tempfile(self, index=None, **kwds): - if index is not None and index in self.open_file_indexes: - self.open_file_indexes.remove(index) - else: - if self.max_open_files: - while len(self.open_file_indexes) >= self.max_open_files: - self.close(self.open_file_indexes[0]) - if index is None: - index = len(self.files) - temp_kwds = dict(self.kwds) - temp_kwds.update(kwds) - # Being able to use delete=True here, would simplify a bit, - # but we support python2.4 in these tools - while True: - try: - tmp_file = tempfile.NamedTemporaryFile(**temp_kwds) - filename = tmp_file.name - break - except OSError as e: - if self.open_file_indexes and e.errno == EMFILE: - self.max_open_files = len(self.open_file_indexes) - self.close(self.open_file_indexes[0]) - else: - raise e - tmp_file.close() - self.files.append(open(filename, 'w+b')) - else: - while True: - try: - self.files[index] = open(self.files[index].name, 'r+b') - break - except OSError as e: - if self.open_file_indexes and e.errno == EMFILE: - self.max_open_files = len(self.open_file_indexes) - self.close(self.open_file_indexes[0]) - else: - raise e - self.files[index].seek(0, 2) - self.open_file_indexes.append(index) - return index, self.files[index] - - def close(self, index, delete=False): - if index in self.open_file_indexes: - self.open_file_indexes.remove(index) - rval = self.files[index].close() - if delete: - try: - os.unlink(self.files[index].name) - except OSError: - pass - return rval - - def flush(self, index): - if index in self.open_file_indexes: - self.files[index].flush() - - def __del__(self): - for i in xrange(len(self.files)): - self.close(i, delete=True) - - -# an object corresponding to a reference layered alignment -class RegionAlignment(object): - - DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca") - MAX_SEQUENCE_SIZE = sys.maxsize # Maximum length of sequence allowed - - def __init__(self, size, species=[], temp_file_handler=None): - assert size <= self.MAX_SEQUENCE_SIZE, "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % (size, self.MAX_SEQUENCE_SIZE) - self.size = size - if not temp_file_handler: - temp_file_handler = TempFileHandler() - self.temp_file_handler = temp_file_handler - self.sequences = {} - if not isinstance(species, list): - species = [species] - for spec in species: - self.add_species(spec) - - # add a species to the alignment - def add_species(self, species): - # make temporary sequence files - file_index, fh = self.temp_file_handler.get_open_tempfile() - self.sequences[species] = file_index - fh.write("-" * self.size) - - # returns the names for species found in alignment, skipping names as requested - def get_species_names(self, skip=[]): - if not isinstance(skip, list): - skip = [skip] - names = list(self.sequences.keys()) - for name in skip: - try: - names.remove(name) - except ValueError: - pass - return names - - # returns the sequence for a species - def get_sequence(self, species): - file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species]) - fh.seek(0) - return fh.read() - - # returns the reverse complement of the sequence for a species - def get_sequence_reverse_complement(self, species): - complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)] - complement.reverse() - return "".join(complement) - - # sets a position for a species - def set_position(self, index, species, base): - if len(base) != 1: - raise Exception("A genomic position can only have a length of 1.") - return self.set_range(index, species, base) - # sets a range for a species - - def set_range(self, index, species, bases): - if index >= self.size or index < 0: - raise Exception("Your index (%i) is out of range (0 - %i)." % (index, self.size - 1)) - if len(bases) == 0: - raise Exception("A set of genomic positions can only have a positive length.") - if species not in self.sequences.keys(): - self.add_species(species) - file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species]) - fh.seek(index) - fh.write(bases) - - # Flush temp file of specified species, or all species - def flush(self, species=None): - if species is None: - species = self.sequences.keys() - elif not isinstance(species, list): - species = [species] - for spec in species: - self.temp_file_handler.flush(self.sequences[spec]) - - -class GenomicRegionAlignment(RegionAlignment): - - def __init__(self, start, end, species=[], temp_file_handler=None): - RegionAlignment.__init__(self, end - start, species, temp_file_handler=temp_file_handler) - self.start = start - self.end = end - - -class SplicedAlignment(object): - - DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca") - - def __init__(self, exon_starts, exon_ends, species=[], temp_file_handler=None): - if not isinstance(exon_starts, list): - exon_starts = [exon_starts] - if not isinstance(exon_ends, list): - exon_ends = [exon_ends] - assert len(exon_starts) == len(exon_ends), "The number of starts does not match the number of sizes." - self.exons = [] - if not temp_file_handler: - temp_file_handler = TempFileHandler() - self.temp_file_handler = temp_file_handler - for i in range(len(exon_starts)): - self.exons.append(GenomicRegionAlignment(exon_starts[i], exon_ends[i], species, temp_file_handler=temp_file_handler)) - - # returns the names for species found in alignment, skipping names as requested - def get_species_names(self, skip=[]): - if not isinstance(skip, list): - skip = [skip] - names = [] - for exon in self.exons: - for name in exon.get_species_names(skip=skip): - if name not in names: - names.append(name) - return names - - # returns the sequence for a species - def get_sequence(self, species): - index, fh = self.temp_file_handler.get_open_tempfile() - for exon in self.exons: - if species in exon.get_species_names(): - seq = exon.get_sequence(species) - # we need to refetch fh here, since exon.get_sequence( species ) uses a tempfile - # and if max==1, it would close fh - index, fh = self.temp_file_handler.get_open_tempfile(index) - fh.write(seq) - else: - fh.write("-" * exon.size) - fh.seek(0) - rval = fh.read() - self.temp_file_handler.close(index, delete=True) - return rval - - # returns the reverse complement of the sequence for a species - def get_sequence_reverse_complement(self, species): - complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)] - complement.reverse() - return "".join(complement) - - # Start and end of coding region - @property - def start(self): - return self.exons[0].start - - @property - def end(self): - return self.exons[-1].end - - -# Open a MAF index using a UID -def maf_index_by_uid(maf_uid, index_location_file): - for line in open(index_location_file): - try: - # read each line, if not enough fields, go to next line - if line[0:1] == "#": - continue - fields = line.split('\t') - if maf_uid == fields[1]: - try: - maf_files = fields[4].replace("\n", "").replace("\r", "").split(",") - return bx.align.maf.MultiIndexed(maf_files, keep_open=True, parse_e_rows=False) - except Exception as e: - raise Exception('MAF UID (%s) found, but configuration appears to be malformed: %s' % (maf_uid, e)) - except Exception: - pass - return None - - -# return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created -def open_or_build_maf_index(maf_file, index_filename, species=None): - try: - return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), None) - except Exception: - return build_maf_index(maf_file, species=species) - - -def build_maf_index_species_chromosomes(filename, index_species=None): - species = [] - species_chromosomes = {} - indexes = bx.interval_index_file.Indexes() - blocks = 0 - try: - maf_reader = bx.align.maf.Reader(open(filename)) - while True: - pos = maf_reader.file.tell() - block = next(maf_reader) - if block is None: - break - blocks += 1 - for c in block.components: - spec = c.src - chrom = None - if "." in spec: - spec, chrom = spec.split(".", 1) - if spec not in species: - species.append(spec) - species_chromosomes[spec] = [] - if chrom and chrom not in species_chromosomes[spec]: - species_chromosomes[spec].append(chrom) - if index_species is None or spec in index_species: - forward_strand_start = c.forward_strand_start - forward_strand_end = c.forward_strand_end - try: - forward_strand_start = int(forward_strand_start) - forward_strand_end = int(forward_strand_end) - except ValueError: - continue # start and end are not integers, can't add component to index, goto next component - # this likely only occurs when parse_e_rows is True? - # could a species exist as only e rows? should the - if forward_strand_end > forward_strand_start: - # require positive length; i.e. certain lines have start = end = 0 and cannot be indexed - indexes.add(c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size) - except Exception as e: - # most likely a bad MAF - log.debug('Building MAF index on %s failed: %s' % (filename, e)) - return (None, [], {}, 0) - return (indexes, species, species_chromosomes, blocks) - - -# builds and returns ( index, index_filename ) for specified maf_file -def build_maf_index(maf_file, species=None): - indexes, found_species, species_chromosomes, blocks = build_maf_index_species_chromosomes(maf_file, species) - if indexes is not None: - fd, index_filename = tempfile.mkstemp() - out = os.fdopen(fd, 'w') - indexes.write(out) - out.close() - return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), index_filename) - return (None, None) - - -def component_overlaps_region(c, region): - if c is None: - return False - start, end = c.get_forward_strand_start(), c.get_forward_strand_end() - if region.start >= end or region.end <= start: - return False - return True - - -def chop_block_by_region(block, src, region, species=None, mincols=0): - # This chopping method was designed to maintain consistency with how start/end padding gaps have been working in Galaxy thus far: - # behavior as seen when forcing blocks to be '+' relative to src sequence (ref) and using block.slice_by_component( ref, slice_start, slice_end ) - # whether-or-not this is the 'correct' behavior is questionable, but this will at least maintain consistency - # comments welcome - slice_start = block.text_size # max for the min() - slice_end = 0 # min for the max() - old_score = block.score # save old score for later use - # We no longer assume only one occurance of src per block, so we need to check them all - for c in iter_components_by_src(block, src): - if component_overlaps_region(c, region): - if c.text is not None: - rev_strand = False - if c.strand == "-": - # We want our coord_to_col coordinates to be returned from positive stranded component - rev_strand = True - c = c.reverse_complement() - start = max(region.start, c.start) - end = min(region.end, c.end) - start = c.coord_to_col(start) - end = c.coord_to_col(end) - if rev_strand: - # need to orient slice coordinates to the original block direction - slice_len = end - start - end = len(c.text) - start - start = end - slice_len - slice_start = min(start, slice_start) - slice_end = max(end, slice_end) - - if slice_start < slice_end: - block = block.slice(slice_start, slice_end) - if block.text_size > mincols: - # restore old score, may not be accurate, but it is better than 0 for everything? - block.score = old_score - if species is not None: - block = block.limit_to_species(species) - block.remove_all_gap_columns() - return block - return None - - -def orient_block_by_region(block, src, region, force_strand=None): - # loop through components matching src, - # make sure each of these components overlap region - # cache strand for each of overlaping regions - # if force_strand / region.strand not in strand cache, reverse complement - # we could have 2 sequences with same src, overlapping region, on different strands, this would cause no reverse_complementing - strands = [c.strand for c in iter_components_by_src(block, src) if component_overlaps_region(c, region)] - if strands and (force_strand is None and region.strand not in strands) or (force_strand is not None and force_strand not in strands): - block = block.reverse_complement() - return block - - -def get_oriented_chopped_blocks_for_region(index, src, region, species=None, mincols=0, force_strand=None): - for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand): - yield block - - -def get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0, force_strand=None): - for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): - yield orient_block_by_region(block, src, region, force_strand), idx, offset - - -# split a block with multiple occurances of src into one block per src -def iter_blocks_split_by_src(block, src): - for src_c in iter_components_by_src(block, src): - new_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) - new_block.text_size = block.text_size - for c in block.components: - if c == src_c or c.src != src: - new_block.add_component(deepcopy(c)) # components have reference to alignment, don't want to lose reference to original alignment block in original components - yield new_block - - -# split a block into multiple blocks with all combinations of a species appearing only once per block -def iter_blocks_split_by_species(block, species=None): - def __split_components_by_species(components_by_species, new_block): - if components_by_species: - # more species with components to add to this block - components_by_species = deepcopy(components_by_species) - spec_comps = components_by_species.pop(0) - for c in spec_comps: - newer_block = deepcopy(new_block) - newer_block.add_component(deepcopy(c)) - for value in __split_components_by_species(components_by_species, newer_block): - yield value - else: - # no more components to add, yield this block - yield new_block - - # divide components by species - spec_dict = {} - if not species: - species = [] - for c in block.components: - spec, chrom = src_split(c.src) - if spec not in spec_dict: - spec_dict[spec] = [] - species.append(spec) - spec_dict[spec].append(c) - else: - for spec in species: - spec_dict[spec] = [] - for c in iter_components_by_src_start(block, spec): - spec_dict[spec].append(c) - - empty_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) # should we copy attributes? - empty_block.text_size = block.text_size - # call recursive function to split into each combo of spec/blocks - for value in __split_components_by_species(list(spec_dict.values()), empty_block): - sort_block_components_by_block(value, block) # restore original component order - yield value - - -# generator yielding only chopped and valid blocks for a specified region -def get_chopped_blocks_for_region(index, src, region, species=None, mincols=0): - for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols): - yield block - - -def get_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0): - for block, idx, offset in index.get_as_iterator_with_index_and_offset(src, region.start, region.end): - block = chop_block_by_region(block, src, region, species, mincols) - if block is not None: - yield block, idx, offset - - -# returns a filled region alignment for specified regions -def get_region_alignment(index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None): - if species is not None: - alignment = RegionAlignment(end - start, species, temp_file_handler=temp_file_handler) - else: - alignment = RegionAlignment(end - start, primary_species, temp_file_handler=temp_file_handler) - return fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps) - - -# reduces a block to only positions exisiting in the src provided -def reduce_block_by_primary_genome(block, species, chromosome, region_start): - # returns ( startIndex, {species:texts} - # where texts' contents are reduced to only positions existing in the primary genome - src = "%s.%s" % (species, chromosome) - ref = block.get_component_by_src(src) - start_offset = ref.start - region_start - species_texts = {} - for c in block.components: - species_texts[c.src.split('.')[0]] = list(c.text) - # remove locations which are gaps in the primary species, starting from the downstream end - for i in range(len(species_texts[species]) - 1, -1, -1): - if species_texts[species][i] == '-': - for text in species_texts.values(): - text.pop(i) - for spec, text in species_texts.items(): - species_texts[spec] = ''.join(text) - return (start_offset, species_texts) - - -# fills a region alignment -def fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True): - region = bx.intervals.Interval(start, end) - region.chrom = chrom - region.strand = strand - primary_src = "%s.%s" % (primary_species, chrom) - - # Order blocks overlaping this position by score, lowest first - blocks = [] - for block, idx, offset in index.get_as_iterator_with_index_and_offset(primary_src, start, end): - score = float(block.score) - for i in range(0, len(blocks)): - if score < blocks[i][0]: - blocks.insert(i, (score, idx, offset)) - break - else: - blocks.append((score, idx, offset)) - - # gap_chars_tuple = tuple( GAP_CHARS ) - gap_chars_str = ''.join(GAP_CHARS) - # Loop through ordered blocks and layer by increasing score - for block_dict in blocks: - for block in iter_blocks_split_by_species(block_dict[1].get_at_offset(block_dict[2])): # need to handle each occurance of sequence in block seperately - if component_overlaps_region(block.get_component_by_src(primary_src), region): - block = chop_block_by_region(block, primary_src, region, species, mincols) # chop block - block = orient_block_by_region(block, primary_src, region) # orient block - start_offset, species_texts = reduce_block_by_primary_genome(block, primary_species, chrom, start) - for spec, text in species_texts.items(): - # we should trim gaps from both sides, since these are not positions in this species genome (sequence) - text = text.rstrip(gap_chars_str) - gap_offset = 0 - # while text.startswith( gap_chars_tuple ): - while True in [text.startswith(gap_char) for gap_char in GAP_CHARS]: # python2.4 doesn't accept a tuple for .startswith() - gap_offset += 1 - text = text[1:] - if not text: - break - if text: - if overwrite_with_gaps: - alignment.set_range(start_offset + gap_offset, spec, text) - else: - for i, char in enumerate(text): - if char not in GAP_CHARS: - alignment.set_position(start_offset + gap_offset + i, spec, char) - return alignment - - -# returns a filled spliced region alignment for specified region with start and end lists -def get_spliced_region_alignment(index, primary_species, chrom, starts, ends, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None): - # create spliced alignment object - if species is not None: - alignment = SplicedAlignment(starts, ends, species, temp_file_handler=temp_file_handler) - else: - alignment = SplicedAlignment(starts, ends, [primary_species], temp_file_handler=temp_file_handler) - for exon in alignment.exons: - fill_region_alignment(exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps) - return alignment - - -# loop through string array, only return non-commented lines -def line_enumerator(lines, comment_start='#'): - i = 0 - for line in lines: - if not line.startswith(comment_start): - i += 1 - yield (i, line) - - -# read a GeneBed file, return list of starts, ends, raw fields -def get_starts_ends_fields_from_gene_bed(line): - # Starts and ends for exons - starts = [] - ends = [] - - fields = line.split() - # Requires atleast 12 BED columns - if len(fields) < 12: - raise Exception("Not a proper 12 column BED line (%s)." % line) - tx_start = int(fields[1]) - strand = fields[5] - if strand != '-': - strand = '+' # Default strand is + - cds_start = int(fields[6]) - cds_end = int(fields[7]) - - # Calculate and store starts and ends of coding exons - region_start, region_end = cds_start, cds_end - exon_starts = list(map(int, fields[11].rstrip(',\n').split(','))) - exon_starts = [x + tx_start for x in exon_starts] - exon_ends = list(map(int, fields[10].rstrip(',').split(','))) - exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)] - for start, end in zip(exon_starts, exon_ends): - start = max(start, region_start) - end = min(end, region_end) - if start < end: - starts.append(start) - ends.append(end) - return (starts, ends, fields) - - -def iter_components_by_src(block, src): - for c in block.components: - if c.src == src: - yield c - - -def get_components_by_src(block, src): - return [value for value in iter_components_by_src(block, src)] - - -def iter_components_by_src_start(block, src): - for c in block.components: - if c.src.startswith(src): - yield c - - -def get_components_by_src_start(block, src): - return [value for value in iter_components_by_src_start(block, src)] - - -def sort_block_components_by_block(block1, block2): - # orders the components in block1 by the index of the component in block2 - # block1 must be a subset of block2 - # occurs in-place - return block1.components.sort(key=functools.cmp_to_key(lambda x, y: block2.components.index(x) - block2.components.index(y))) - - -def get_species_in_maf(maf_filename): - species = [] - for block in bx.align.maf.Reader(open(maf_filename)): - for spec in get_species_in_block(block): - if spec not in species: - species.append(spec) - return species - - -def parse_species_option(species): - if species: - species = species.split(',') - if 'None' not in species: - return species - return None # provided species was '', None, or had 'None' in it - - -def remove_temp_index_file(index_filename): - try: - os.unlink(index_filename) - except Exception: - pass - -# Below are methods to deal with FASTA files - - -def get_fasta_header(component, attributes={}, suffix=None): - header = ">%s(%s):%i-%i|" % (component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end()) - for key, value in attributes.items(): - header = "%s%s=%s|" % (header, key, value) - if suffix: - header = "%s%s" % (header, suffix) - else: - header = "%s%s" % (header, src_split(component.src)[0]) - return header - - -def get_attributes_from_fasta_header(header): - if not header: - return {} - attributes = {} - header = header.lstrip('>') - header = header.strip() - fields = header.split('|') - try: - region = fields[0] - region = region.split('(', 1) - temp = region[0].split('.', 1) - attributes['species'] = temp[0] - if len(temp) == 2: - attributes['chrom'] = temp[1] - else: - attributes['chrom'] = temp[0] - region = region[1].split(')', 1) - attributes['strand'] = region[0] - region = region[1].lstrip(':').split('-') - attributes['start'] = int(region[0]) - attributes['end'] = int(region[1]) - except Exception: - # fields 0 is not a region coordinate - pass - if len(fields) > 2: - for i in range(1, len(fields) - 1): - prop = fields[i].split('=', 1) - if len(prop) == 2: - attributes[prop[0]] = prop[1] - if len(fields) > 1: - attributes['__suffix__'] = fields[-1] - return attributes - - -def iter_fasta_alignment(filename): - class fastaComponent(object): - def __init__(self, species, text=""): - self.species = species - self.text = text - - def extend(self, text): - self.text = self.text + text.replace('\n', '').replace('\r', '').strip() - # yields a list of fastaComponents for a FASTA file - f = open(filename, 'rb') - components = [] - # cur_component = None - while True: - line = f.readline() - if not line: - if components: - yield components - return - line = line.strip() - if not line: - if components: - yield components - components = [] - elif line.startswith('>'): - attributes = get_attributes_from_fasta_header(line) - components.append(fastaComponent(attributes['species'])) - elif components: - components[-1].extend(line) diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py new file mode 120000 index 00000000000..07bafe2d1a5 --- /dev/null +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -0,0 +1 @@ +../../datatypes/util/maf_utilities.py \ No newline at end of file diff --git a/lib/galaxy/util/tool_shed/__init__.py b/lib/galaxy/util/tool_shed/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/util/tool_shed/common_util.py b/lib/galaxy/util/tool_shed/common_util.py new file mode 100644 index 00000000000..41ac1cf1534 --- /dev/null +++ b/lib/galaxy/util/tool_shed/common_util.py @@ -0,0 +1,377 @@ +import errno +import json +import logging +import os + +from routes import url_for +from six.moves.urllib.parse import urljoin + +from galaxy import util +from galaxy.util.odict import odict +from galaxy.util.tool_shed import encoding_util, xml_util + +log = logging.getLogger(__name__) + +REPOSITORY_OWNER = 'devteam' +TOOL_MIGRATION_SCRIPTS_DIR = os.path.abspath(os.path.join( + os.path.dirname(__file__), os.pardir, 'galaxy_install', 'migrate', 'scripts')) +TOOL_MIGRATION_VERSIONS_DIR = os.path.abspath(os.path.join( + os.path.dirname(__file__), os.pardir, 'galaxy_install', 'migrate', 'versions')) + + +def accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies): + if tool_shed_accessible: + if tool_dependencies: + for tool_dependency in tool_dependencies: + if tool_dependency not in all_tool_dependencies: + all_tool_dependencies.append(tool_dependency) + return all_tool_dependencies + + +def check_for_missing_tools(app, tool_panel_configs, latest_tool_migration_script_number): + # Get the 000x_tools.xml file associated with the current migrate_tools version number. + tools_xml_file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, 'galaxy_install', + 'migrate', 'scripts', + '%04d_tools.xml' % latest_tool_migration_script_number)) + # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. + migrated_tool_configs_dict = odict() + tree, error_message = xml_util.parse_xml(tools_xml_file_path) + if tree is None: + return False, odict() + root = tree.getroot() + tool_shed = root.get('name') + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) + # The default behavior is that the tool shed is down. + tool_shed_accessible = False + missing_tool_configs_dict = odict() + if tool_shed_url: + for elem in root: + if elem.tag == 'repository': + repository_dependencies = [] + all_tool_dependencies = [] + repository_name = elem.get('name') + changeset_revision = elem.get('changeset_revision') + tool_shed_accessible, repository_dependencies_dict = get_repository_dependencies(app, + tool_shed_url, + repository_name, + REPOSITORY_OWNER, + changeset_revision) + if tool_shed_accessible: + # Accumulate all tool dependencies defined for repository dependencies for display to the user. + for rd_key, rd_tups in repository_dependencies_dict.items(): + if rd_key in ['root_key', 'description']: + continue + for rd_tup in rd_tups: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + parse_repository_dependency_tuple(rd_tup) + tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, + tool_shed_url, + name, + owner, + changeset_revision) + all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) + tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, + tool_shed_url, + repository_name, + REPOSITORY_OWNER, + changeset_revision) + all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) + for tool_elem in elem.findall('tool'): + tool_config_file_name = tool_elem.get('file') + if tool_config_file_name: + # We currently do nothing with repository dependencies except install them (we do not display repositories that will be + # installed to the user). However, we'll store them in the following dictionary in case we choose to display them in the + # future. + dependencies_dict = dict(tool_dependencies=all_tool_dependencies, + repository_dependencies=repository_dependencies) + migrated_tool_configs_dict[tool_config_file_name] = dependencies_dict + else: + break + if tool_shed_accessible: + # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. + for tool_panel_config in tool_panel_configs: + tree, error_message = xml_util.parse_xml(tool_panel_config) + if tree: + root = tree.getroot() + for elem in root: + if elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict) + elif elem.tag == 'section': + for section_elem in elem: + if section_elem.tag == 'tool': + missing_tool_configs_dict = check_tool_tag_set(section_elem, migrated_tool_configs_dict, missing_tool_configs_dict) + else: + exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % (tool_shed, app.config.tool_sheds_config) + exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' + raise Exception(exception_msg) + return tool_shed_accessible, missing_tool_configs_dict + + +def check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict): + file_path = elem.get('file', None) + if file_path: + name = os.path.basename(file_path) + for migrated_tool_config in migrated_tool_configs_dict.keys(): + if migrated_tool_config in [file_path, name]: + missing_tool_configs_dict[name] = migrated_tool_configs_dict[migrated_tool_config] + return missing_tool_configs_dict + + +def generate_clone_url_for_installed_repository(app, repository): + """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, str(repository.tool_shed)) + return util.build_url(tool_shed_url, pathspec=['repos', str(repository.owner), str(repository.name)]) + + +def generate_clone_url_for_repository_in_tool_shed(user, repository): + """Generate the URL for cloning a repository that is in the tool shed.""" + base_url = url_for('/', qualified=True).rstrip('/') + if user: + protocol, base = base_url.split('://') + username = '%s@' % user.username + return '%s://%s%s/repos/%s/%s' % (protocol, username, base, repository.user.username, repository.name) + else: + return '%s/repos/%s/%s' % (base_url, repository.user.username, repository.name) + + +def generate_clone_url_from_repo_info_tup(app, repo_info_tup): + """Generate the URL for cloning a repository given a tuple of toolshed, name, owner, changeset_revision.""" + # Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab', False] + toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ + parse_repository_dependency_tuple(repo_info_tup) + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, toolshed) + # Don't include the changeset_revision in clone urls. + return util.build_url(tool_shed_url, pathspec=['repos', owner, name]) + + +def get_non_shed_tool_panel_configs(app): + """Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml.""" + config_filenames = [] + for config_filename in app.config.tool_configs: + # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. + # + try: + tree, error_message = xml_util.parse_xml(config_filename) + except (OSError, IOError) as exc: + if (config_filename == app.config.shed_tool_conf and not + app.config.shed_tool_conf_set and + exc.errno == errno.ENOENT): + continue + raise + if tree is None: + continue + root = tree.getroot() + tool_path = root.get('tool_path', None) + if tool_path is None: + config_filenames.append(config_filename) + return config_filenames + + +def get_repository_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): + repository_dependencies_dict = {} + tool_shed_accessible = True + params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) + pathspec = ['repository', 'get_repository_dependencies'] + try: + raw_text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) + tool_shed_accessible = True + except Exception as e: + tool_shed_accessible = False + log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) + if tool_shed_accessible: + if len(raw_text) > 2: + encoded_text = json.loads(util.unicodify(raw_text)) + repository_dependencies_dict = encoding_util.tool_shed_decode(encoded_text) + return tool_shed_accessible, repository_dependencies_dict + + +def get_protocol_from_tool_shed_url(tool_shed_url): + """Return the protocol from the received tool_shed_url if it exists.""" + try: + if tool_shed_url.find('://') > 0: + return tool_shed_url.split('://')[0].lower() + except Exception: + # We receive a lot of calls here where the tool_shed_url is None. The container_util uses + # that value when creating a header row. If the tool_shed_url is not None, we have a problem. + if tool_shed_url is not None: + log.exception("Handled exception getting the protocol from Tool Shed URL %s", str(tool_shed_url)) + # Default to HTTP protocol. + return 'http' + + +def get_tool_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): + tool_dependencies = [] + tool_shed_accessible = True + params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) + pathspec = ['repository', 'get_tool_dependencies'] + try: + text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) + tool_shed_accessible = True + except Exception as e: + tool_shed_accessible = False + log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) + if tool_shed_accessible: + if text: + tool_dependencies_dict = encoding_util.tool_shed_decode(text) + for requirements_dict in tool_dependencies_dict.values(): + tool_dependency_name = requirements_dict['name'] + tool_dependency_version = requirements_dict['version'] + tool_dependency_type = requirements_dict['type'] + tool_dependencies.append((tool_dependency_name, tool_dependency_version, tool_dependency_type)) + return tool_shed_accessible, tool_dependencies + + +def get_tool_shed_repository_ids(as_string=False, **kwd): + tsrid = kwd.get('tool_shed_repository_id', None) + tsridslist = util.listify(kwd.get('tool_shed_repository_ids', None)) + if not tsridslist: + tsridslist = util.listify(kwd.get('id', None)) + if tsridslist is not None: + if tsrid is not None and tsrid not in tsridslist: + tsridslist.append(tsrid) + if as_string: + return ','.join(tsridslist) + return tsridslist + else: + tsridslist = util.listify(kwd.get('ordered_tsr_ids', None)) + if tsridslist is not None: + if as_string: + return ','.join(tsridslist) + return tsridslist + if as_string: + return '' + return [] + + +def get_tool_shed_url_from_tool_shed_registry(app, tool_shed): + """ + The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is + something like: http://toolshed.g2.bx.psu.edu/ + """ + cleaned_tool_shed = remove_protocol_from_tool_shed_url(tool_shed) + for shed_url in app.tool_shed_registry.tool_sheds.values(): + if shed_url.find(cleaned_tool_shed) >= 0: + if shed_url.endswith('/'): + shed_url = shed_url.rstrip('/') + return shed_url + # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. + return None + + +def get_tool_shed_repository_url(app, tool_shed, owner, name): + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) + if tool_shed_url: + # Append a slash to the tool shed URL, because urlparse.urljoin will eliminate + # the last part of a URL if it does not end with a forward slash. + tool_shed_url = '%s/' % tool_shed_url + return urljoin(tool_shed_url, 'view/%s/%s' % (owner, name)) + return tool_shed_url + + +def get_user_by_username(app, username): + """Get a user from the database by username.""" + sa_session = app.model.context.current + try: + user = sa_session.query(app.model.User) \ + .filter(app.model.User.table.c.username == username) \ + .one() + return user + except Exception: + return None + + +def handle_galaxy_url(trans, **kwd): + galaxy_url = kwd.get('galaxy_url', None) + if galaxy_url: + trans.set_cookie(galaxy_url, name='toolshedgalaxyurl') + else: + galaxy_url = trans.get_cookie(name='toolshedgalaxyurl') + return galaxy_url + + +def handle_tool_shed_url_protocol(app, shed_url): + """Handle secure and insecure HTTP protocol since they may change over time.""" + try: + if app.name == 'galaxy': + url = remove_protocol_from_tool_shed_url(shed_url) + tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, url) + else: + tool_shed_url = str(url_for('/', qualified=True)).rstrip('/') + return tool_shed_url + except Exception: + # We receive a lot of calls here where the tool_shed_url is None. The container_util uses + # that value when creating a header row. If the tool_shed_url is not None, we have a problem. + if shed_url is not None: + log.exception("Handled exception removing protocol from URL %s", str(shed_url)) + return shed_url + + +def parse_repository_dependency_tuple(repository_dependency_tuple, contains_error=False): + # Default both prior_installation_required and only_if_compiling_contained_td to False in cases where metadata should be reset on the + # repository containing the repository_dependency definition. + prior_installation_required = 'False' + only_if_compiling_contained_td = 'False' + if contains_error: + if len(repository_dependency_tuple) == 5: + tool_shed, name, owner, changeset_revision, error = repository_dependency_tuple + elif len(repository_dependency_tuple) == 6: + tool_shed, name, owner, changeset_revision, prior_installation_required, error = repository_dependency_tuple + elif len(repository_dependency_tuple) == 7: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \ + repository_dependency_tuple + return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error + else: + if len(repository_dependency_tuple) == 4: + tool_shed, name, owner, changeset_revision = repository_dependency_tuple + elif len(repository_dependency_tuple) == 5: + tool_shed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tuple + elif len(repository_dependency_tuple) == 6: + tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tuple + return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td + + +def remove_port_from_tool_shed_url(tool_shed_url): + """Return a partial Tool Shed URL, eliminating the port if it exists.""" + try: + if tool_shed_url.find(':') > 0: + # Eliminate the port, if any, since it will result in an invalid directory name. + new_tool_shed_url = tool_shed_url.split(':')[0] + else: + new_tool_shed_url = tool_shed_url + return new_tool_shed_url.rstrip('/') + except Exception: + # We receive a lot of calls here where the tool_shed_url is None. The container_util uses + # that value when creating a header row. If the tool_shed_url is not None, we have a problem. + if tool_shed_url is not None: + log.exception("Handled exception removing the port from Tool Shed URL %s", str(tool_shed_url)) + return tool_shed_url + + +def remove_protocol_and_port_from_tool_shed_url(tool_shed_url): + """Return a partial Tool Shed URL, eliminating the protocol and/or port if either exists.""" + tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url) + tool_shed = remove_port_from_tool_shed_url(tool_shed) + return tool_shed + + +def remove_protocol_and_user_from_clone_url(repository_clone_url): + """Return a URL that can be used to clone a repository, eliminating the protocol and user if either exists.""" + if repository_clone_url.find('@') > 0: + # We have an url that includes an authenticated user, something like: + # http://test@bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split('@') + tmp_url = items[1] + elif repository_clone_url.find('//') > 0: + # We have an url that includes only a protocol, something like: + # http://bx.psu.edu:9009/repos/some_username/column + items = repository_clone_url.split('//') + tmp_url = items[1] + else: + tmp_url = repository_clone_url + return tmp_url.rstrip('/') + + +def remove_protocol_from_tool_shed_url(tool_shed_url): + """Return a partial Tool Shed URL, eliminating the protocol if it exists.""" + return util.remove_protocol_from_url(tool_shed_url) diff --git a/lib/galaxy/util/tool_shed/encoding_util.py b/lib/galaxy/util/tool_shed/encoding_util.py new file mode 100644 index 00000000000..57c1a11c8a7 --- /dev/null +++ b/lib/galaxy/util/tool_shed/encoding_util.py @@ -0,0 +1,41 @@ +import binascii +import json + +from galaxy.util import ( + smart_str, + unicodify +) +from galaxy.util.hash_util import hmac_new + + +encoding_sep = '__esep__' +encoding_sep2 = '__esepii__' + + +def tool_shed_decode(value): + # Extract and verify hash + value = unicodify(value) + a, b = value.split(":") + value = binascii.unhexlify(b) + test = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', value) + assert a == test + # Restore from string + values = None + value = unicodify(value) + try: + values = json.loads(value) + except Exception: + pass + if values is None: + values = value + return values + + +def tool_shed_encode(val): + if isinstance(val, dict) or isinstance(val, list): + value = json.dumps(val) + else: + value = val + a = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', smart_str(value)) + b = unicodify(binascii.hexlify(smart_str(value))) + return "%s:%s" % (a, b) diff --git a/lib/galaxy/util/tool_shed/xml_util.py b/lib/galaxy/util/tool_shed/xml_util.py new file mode 100644 index 00000000000..5e7e9db05e4 --- /dev/null +++ b/lib/galaxy/util/tool_shed/xml_util.py @@ -0,0 +1,92 @@ +import io +import logging +import os +import tempfile +from xml.etree import ElementTree as XmlET + +from galaxy.util import ( + xml_to_string +) + +log = logging.getLogger(__name__) + + +class Py27CommentedTreeBuilder(XmlET.TreeBuilder): + + def doctype(*args): + # handle deprecation warning for XMLParsing a file with DOCTYPE + pass + + def comment(self, data): + self.start(XmlET.Comment, {}) + self.data(data) + self.end(XmlET.Comment) + + +def create_and_write_tmp_file(elem): + tmp_str = xml_to_string(elem, pretty=True) + with tempfile.NamedTemporaryFile(prefix="tmp-toolshed-cawrf", delete=False) as fh: + tmp_filename = fh.name + with io.open(tmp_filename, mode='w', encoding='utf-8') as fh: + fh.write(tmp_str) + return tmp_filename + + +def create_element(tag, attributes=None, sub_elements=None): + """ + Create a new element whose tag is the value of the received tag, and whose attributes are all + key / value pairs in the received attributes and sub_elements. + """ + if tag: + elem = XmlET.Element(tag) + if attributes: + # The received attributes is an odict to preserve ordering. + for k, v in attributes.items(): + elem.set(k, v) + if sub_elements: + # The received attributes is an odict. These handle information that tends to be + # long text including paragraphs (e.g., description and long_description. + for k, v in sub_elements.items(): + # Don't include fields that are blank. + if v: + if k == 'packages': + # The received sub_elements is an odict whose key is 'packages' and whose + # value is a list of ( name, version ) tuples. + for v_tuple in v: + sub_elem = XmlET.SubElement(elem, 'package') + sub_elem_name, sub_elem_version = v_tuple + sub_elem.set('name', sub_elem_name) + sub_elem.set('version', sub_elem_version) + elif isinstance(v, list): + sub_elem = XmlET.SubElement(elem, k) + # If v is a list, then it must be a list of tuples where the first + # item is the tag and the second item is the text value. + for v_tuple in v: + if len(v_tuple) == 2: + v_tag = v_tuple[0] + v_text = v_tuple[1] + # Don't include fields that are blank. + if v_text: + v_elem = XmlET.SubElement(sub_elem, v_tag) + v_elem.text = v_text + else: + sub_elem = XmlET.SubElement(elem, k) + sub_elem.text = v + return elem + return None + + +def parse_xml(file_name): + """Returns a parsed xml tree with comments intact.""" + error_message = '' + if not os.path.exists(file_name): + return None, "File does not exist %s" % str(file_name) + + with open(file_name, 'r') as fobj: + try: + tree = XmlET.parse(fobj, parser=XmlET.XMLParser(target=Py27CommentedTreeBuilder())) + except Exception as e: + error_message = "Exception attempting to parse %s: %s" % (str(file_name), str(e)) + log.exception(error_message) + return None, error_message + return tree, error_message diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py deleted file mode 100644 index 7caeb00f18b..00000000000 --- a/lib/tool_shed/util/common_util.py +++ /dev/null @@ -1,377 +0,0 @@ -import errno -import json -import logging -import os - -from six.moves.urllib.parse import urljoin - -from galaxy import util -from galaxy.util.odict import odict -from galaxy.web import url_for -from tool_shed.util import encoding_util, xml_util - -log = logging.getLogger(__name__) - -REPOSITORY_OWNER = 'devteam' -TOOL_MIGRATION_SCRIPTS_DIR = os.path.abspath(os.path.join( - os.path.dirname(__file__), os.pardir, 'galaxy_install', 'migrate', 'scripts')) -TOOL_MIGRATION_VERSIONS_DIR = os.path.abspath(os.path.join( - os.path.dirname(__file__), os.pardir, 'galaxy_install', 'migrate', 'versions')) - - -def accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies): - if tool_shed_accessible: - if tool_dependencies: - for tool_dependency in tool_dependencies: - if tool_dependency not in all_tool_dependencies: - all_tool_dependencies.append(tool_dependency) - return all_tool_dependencies - - -def check_for_missing_tools(app, tool_panel_configs, latest_tool_migration_script_number): - # Get the 000x_tools.xml file associated with the current migrate_tools version number. - tools_xml_file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), - os.pardir, 'galaxy_install', - 'migrate', 'scripts', - '%04d_tools.xml' % latest_tool_migration_script_number)) - # Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config. - migrated_tool_configs_dict = odict() - tree, error_message = xml_util.parse_xml(tools_xml_file_path) - if tree is None: - return False, odict() - root = tree.getroot() - tool_shed = root.get('name') - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) - # The default behavior is that the tool shed is down. - tool_shed_accessible = False - missing_tool_configs_dict = odict() - if tool_shed_url: - for elem in root: - if elem.tag == 'repository': - repository_dependencies = [] - all_tool_dependencies = [] - repository_name = elem.get('name') - changeset_revision = elem.get('changeset_revision') - tool_shed_accessible, repository_dependencies_dict = get_repository_dependencies(app, - tool_shed_url, - repository_name, - REPOSITORY_OWNER, - changeset_revision) - if tool_shed_accessible: - # Accumulate all tool dependencies defined for repository dependencies for display to the user. - for rd_key, rd_tups in repository_dependencies_dict.items(): - if rd_key in ['root_key', 'description']: - continue - for rd_tup in rd_tups: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - parse_repository_dependency_tuple(rd_tup) - tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, - tool_shed_url, - name, - owner, - changeset_revision) - all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) - tool_shed_accessible, tool_dependencies = get_tool_dependencies(app, - tool_shed_url, - repository_name, - REPOSITORY_OWNER, - changeset_revision) - all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies) - for tool_elem in elem.findall('tool'): - tool_config_file_name = tool_elem.get('file') - if tool_config_file_name: - # We currently do nothing with repository dependencies except install them (we do not display repositories that will be - # installed to the user). However, we'll store them in the following dictionary in case we choose to display them in the - # future. - dependencies_dict = dict(tool_dependencies=all_tool_dependencies, - repository_dependencies=repository_dependencies) - migrated_tool_configs_dict[tool_config_file_name] = dependencies_dict - else: - break - if tool_shed_accessible: - # Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names. - for tool_panel_config in tool_panel_configs: - tree, error_message = xml_util.parse_xml(tool_panel_config) - if tree: - root = tree.getroot() - for elem in root: - if elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict) - elif elem.tag == 'section': - for section_elem in elem: - if section_elem.tag == 'tool': - missing_tool_configs_dict = check_tool_tag_set(section_elem, migrated_tool_configs_dict, missing_tool_configs_dict) - else: - exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % (tool_shed, app.config.tool_sheds_config) - exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' - raise Exception(exception_msg) - return tool_shed_accessible, missing_tool_configs_dict - - -def check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict): - file_path = elem.get('file', None) - if file_path: - name = os.path.basename(file_path) - for migrated_tool_config in migrated_tool_configs_dict.keys(): - if migrated_tool_config in [file_path, name]: - missing_tool_configs_dict[name] = migrated_tool_configs_dict[migrated_tool_config] - return missing_tool_configs_dict - - -def generate_clone_url_for_installed_repository(app, repository): - """Generate the URL for cloning a repository that has been installed into a Galaxy instance.""" - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, str(repository.tool_shed)) - return util.build_url(tool_shed_url, pathspec=['repos', str(repository.owner), str(repository.name)]) - - -def generate_clone_url_for_repository_in_tool_shed(user, repository): - """Generate the URL for cloning a repository that is in the tool shed.""" - base_url = url_for('/', qualified=True).rstrip('/') - if user: - protocol, base = base_url.split('://') - username = '%s@' % user.username - return '%s://%s%s/repos/%s/%s' % (protocol, username, base, repository.user.username, repository.name) - else: - return '%s/repos/%s/%s' % (base_url, repository.user.username, repository.name) - - -def generate_clone_url_from_repo_info_tup(app, repo_info_tup): - """Generate the URL for cloning a repository given a tuple of toolshed, name, owner, changeset_revision.""" - # Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab', False] - toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \ - parse_repository_dependency_tuple(repo_info_tup) - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, toolshed) - # Don't include the changeset_revision in clone urls. - return util.build_url(tool_shed_url, pathspec=['repos', owner, name]) - - -def get_non_shed_tool_panel_configs(app): - """Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml.""" - config_filenames = [] - for config_filename in app.config.tool_configs: - # Any config file that includes a tool_path attribute in the root tag set like the following is shed-related. - # - try: - tree, error_message = xml_util.parse_xml(config_filename) - except (OSError, IOError) as exc: - if (config_filename == app.config.shed_tool_conf and not - app.config.shed_tool_conf_set and - exc.errno == errno.ENOENT): - continue - raise - if tree is None: - continue - root = tree.getroot() - tool_path = root.get('tool_path', None) - if tool_path is None: - config_filenames.append(config_filename) - return config_filenames - - -def get_repository_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): - repository_dependencies_dict = {} - tool_shed_accessible = True - params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) - pathspec = ['repository', 'get_repository_dependencies'] - try: - raw_text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) - tool_shed_accessible = True - except Exception as e: - tool_shed_accessible = False - log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) - if tool_shed_accessible: - if len(raw_text) > 2: - encoded_text = json.loads(util.unicodify(raw_text)) - repository_dependencies_dict = encoding_util.tool_shed_decode(encoded_text) - return tool_shed_accessible, repository_dependencies_dict - - -def get_protocol_from_tool_shed_url(tool_shed_url): - """Return the protocol from the received tool_shed_url if it exists.""" - try: - if tool_shed_url.find('://') > 0: - return tool_shed_url.split('://')[0].lower() - except Exception: - # We receive a lot of calls here where the tool_shed_url is None. The container_util uses - # that value when creating a header row. If the tool_shed_url is not None, we have a problem. - if tool_shed_url is not None: - log.exception("Handled exception getting the protocol from Tool Shed URL %s", str(tool_shed_url)) - # Default to HTTP protocol. - return 'http' - - -def get_tool_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision): - tool_dependencies = [] - tool_shed_accessible = True - params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision) - pathspec = ['repository', 'get_tool_dependencies'] - try: - text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params) - tool_shed_accessible = True - except Exception as e: - tool_shed_accessible = False - log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e) - if tool_shed_accessible: - if text: - tool_dependencies_dict = encoding_util.tool_shed_decode(text) - for requirements_dict in tool_dependencies_dict.values(): - tool_dependency_name = requirements_dict['name'] - tool_dependency_version = requirements_dict['version'] - tool_dependency_type = requirements_dict['type'] - tool_dependencies.append((tool_dependency_name, tool_dependency_version, tool_dependency_type)) - return tool_shed_accessible, tool_dependencies - - -def get_tool_shed_repository_ids(as_string=False, **kwd): - tsrid = kwd.get('tool_shed_repository_id', None) - tsridslist = util.listify(kwd.get('tool_shed_repository_ids', None)) - if not tsridslist: - tsridslist = util.listify(kwd.get('id', None)) - if tsridslist is not None: - if tsrid is not None and tsrid not in tsridslist: - tsridslist.append(tsrid) - if as_string: - return ','.join(tsridslist) - return tsridslist - else: - tsridslist = util.listify(kwd.get('ordered_tsr_ids', None)) - if tsridslist is not None: - if as_string: - return ','.join(tsridslist) - return tsridslist - if as_string: - return '' - return [] - - -def get_tool_shed_url_from_tool_shed_registry(app, tool_shed): - """ - The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is - something like: http://toolshed.g2.bx.psu.edu/ - """ - cleaned_tool_shed = remove_protocol_from_tool_shed_url(tool_shed) - for shed_url in app.tool_shed_registry.tool_sheds.values(): - if shed_url.find(cleaned_tool_shed) >= 0: - if shed_url.endswith('/'): - shed_url = shed_url.rstrip('/') - return shed_url - # The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml. - return None - - -def get_tool_shed_repository_url(app, tool_shed, owner, name): - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed) - if tool_shed_url: - # Append a slash to the tool shed URL, because urlparse.urljoin will eliminate - # the last part of a URL if it does not end with a forward slash. - tool_shed_url = '%s/' % tool_shed_url - return urljoin(tool_shed_url, 'view/%s/%s' % (owner, name)) - return tool_shed_url - - -def get_user_by_username(app, username): - """Get a user from the database by username.""" - sa_session = app.model.context.current - try: - user = sa_session.query(app.model.User) \ - .filter(app.model.User.table.c.username == username) \ - .one() - return user - except Exception: - return None - - -def handle_galaxy_url(trans, **kwd): - galaxy_url = kwd.get('galaxy_url', None) - if galaxy_url: - trans.set_cookie(galaxy_url, name='toolshedgalaxyurl') - else: - galaxy_url = trans.get_cookie(name='toolshedgalaxyurl') - return galaxy_url - - -def handle_tool_shed_url_protocol(app, shed_url): - """Handle secure and insecure HTTP protocol since they may change over time.""" - try: - if app.name == 'galaxy': - url = remove_protocol_from_tool_shed_url(shed_url) - tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, url) - else: - tool_shed_url = str(url_for('/', qualified=True)).rstrip('/') - return tool_shed_url - except Exception: - # We receive a lot of calls here where the tool_shed_url is None. The container_util uses - # that value when creating a header row. If the tool_shed_url is not None, we have a problem. - if shed_url is not None: - log.exception("Handled exception removing protocol from URL %s", str(shed_url)) - return shed_url - - -def parse_repository_dependency_tuple(repository_dependency_tuple, contains_error=False): - # Default both prior_installation_required and only_if_compiling_contained_td to False in cases where metadata should be reset on the - # repository containing the repository_dependency definition. - prior_installation_required = 'False' - only_if_compiling_contained_td = 'False' - if contains_error: - if len(repository_dependency_tuple) == 5: - tool_shed, name, owner, changeset_revision, error = repository_dependency_tuple - elif len(repository_dependency_tuple) == 6: - tool_shed, name, owner, changeset_revision, prior_installation_required, error = repository_dependency_tuple - elif len(repository_dependency_tuple) == 7: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \ - repository_dependency_tuple - return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error - else: - if len(repository_dependency_tuple) == 4: - tool_shed, name, owner, changeset_revision = repository_dependency_tuple - elif len(repository_dependency_tuple) == 5: - tool_shed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tuple - elif len(repository_dependency_tuple) == 6: - tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tuple - return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td - - -def remove_port_from_tool_shed_url(tool_shed_url): - """Return a partial Tool Shed URL, eliminating the port if it exists.""" - try: - if tool_shed_url.find(':') > 0: - # Eliminate the port, if any, since it will result in an invalid directory name. - new_tool_shed_url = tool_shed_url.split(':')[0] - else: - new_tool_shed_url = tool_shed_url - return new_tool_shed_url.rstrip('/') - except Exception: - # We receive a lot of calls here where the tool_shed_url is None. The container_util uses - # that value when creating a header row. If the tool_shed_url is not None, we have a problem. - if tool_shed_url is not None: - log.exception("Handled exception removing the port from Tool Shed URL %s", str(tool_shed_url)) - return tool_shed_url - - -def remove_protocol_and_port_from_tool_shed_url(tool_shed_url): - """Return a partial Tool Shed URL, eliminating the protocol and/or port if either exists.""" - tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url) - tool_shed = remove_port_from_tool_shed_url(tool_shed) - return tool_shed - - -def remove_protocol_and_user_from_clone_url(repository_clone_url): - """Return a URL that can be used to clone a repository, eliminating the protocol and user if either exists.""" - if repository_clone_url.find('@') > 0: - # We have an url that includes an authenticated user, something like: - # http://test@bx.psu.edu:9009/repos/some_username/column - items = repository_clone_url.split('@') - tmp_url = items[1] - elif repository_clone_url.find('//') > 0: - # We have an url that includes only a protocol, something like: - # http://bx.psu.edu:9009/repos/some_username/column - items = repository_clone_url.split('//') - tmp_url = items[1] - else: - tmp_url = repository_clone_url - return tmp_url.rstrip('/') - - -def remove_protocol_from_tool_shed_url(tool_shed_url): - """Return a partial Tool Shed URL, eliminating the protocol if it exists.""" - return util.remove_protocol_from_url(tool_shed_url) diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py new file mode 120000 index 00000000000..71148b57d1c --- /dev/null +++ b/lib/tool_shed/util/common_util.py @@ -0,0 +1 @@ +../../galaxy/util/tool_shed/common_util.py \ No newline at end of file diff --git a/lib/tool_shed/util/encoding_util.py b/lib/tool_shed/util/encoding_util.py deleted file mode 100644 index f5bb2ff6441..00000000000 --- a/lib/tool_shed/util/encoding_util.py +++ /dev/null @@ -1,43 +0,0 @@ -import binascii -import json -import logging - -from galaxy.util import ( - smart_str, - unicodify -) -from galaxy.util.hash_util import hmac_new - -log = logging.getLogger(__name__) - -encoding_sep = '__esep__' -encoding_sep2 = '__esepii__' - - -def tool_shed_decode(value): - # Extract and verify hash - value = unicodify(value) - a, b = value.split(":") - value = binascii.unhexlify(b) - test = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', value) - assert a == test - # Restore from string - values = None - value = unicodify(value) - try: - values = json.loads(value) - except Exception: - pass - if values is None: - values = value - return values - - -def tool_shed_encode(val): - if isinstance(val, dict) or isinstance(val, list): - value = json.dumps(val) - else: - value = val - a = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', smart_str(value)) - b = unicodify(binascii.hexlify(smart_str(value))) - return "%s:%s" % (a, b) diff --git a/lib/tool_shed/util/encoding_util.py b/lib/tool_shed/util/encoding_util.py new file mode 120000 index 00000000000..5bb58723d83 --- /dev/null +++ b/lib/tool_shed/util/encoding_util.py @@ -0,0 +1 @@ +../../galaxy/util/tool_shed/encoding_util.py \ No newline at end of file diff --git a/lib/tool_shed/util/xml_util.py b/lib/tool_shed/util/xml_util.py deleted file mode 100644 index 5e7e9db05e4..00000000000 --- a/lib/tool_shed/util/xml_util.py +++ /dev/null @@ -1,92 +0,0 @@ -import io -import logging -import os -import tempfile -from xml.etree import ElementTree as XmlET - -from galaxy.util import ( - xml_to_string -) - -log = logging.getLogger(__name__) - - -class Py27CommentedTreeBuilder(XmlET.TreeBuilder): - - def doctype(*args): - # handle deprecation warning for XMLParsing a file with DOCTYPE - pass - - def comment(self, data): - self.start(XmlET.Comment, {}) - self.data(data) - self.end(XmlET.Comment) - - -def create_and_write_tmp_file(elem): - tmp_str = xml_to_string(elem, pretty=True) - with tempfile.NamedTemporaryFile(prefix="tmp-toolshed-cawrf", delete=False) as fh: - tmp_filename = fh.name - with io.open(tmp_filename, mode='w', encoding='utf-8') as fh: - fh.write(tmp_str) - return tmp_filename - - -def create_element(tag, attributes=None, sub_elements=None): - """ - Create a new element whose tag is the value of the received tag, and whose attributes are all - key / value pairs in the received attributes and sub_elements. - """ - if tag: - elem = XmlET.Element(tag) - if attributes: - # The received attributes is an odict to preserve ordering. - for k, v in attributes.items(): - elem.set(k, v) - if sub_elements: - # The received attributes is an odict. These handle information that tends to be - # long text including paragraphs (e.g., description and long_description. - for k, v in sub_elements.items(): - # Don't include fields that are blank. - if v: - if k == 'packages': - # The received sub_elements is an odict whose key is 'packages' and whose - # value is a list of ( name, version ) tuples. - for v_tuple in v: - sub_elem = XmlET.SubElement(elem, 'package') - sub_elem_name, sub_elem_version = v_tuple - sub_elem.set('name', sub_elem_name) - sub_elem.set('version', sub_elem_version) - elif isinstance(v, list): - sub_elem = XmlET.SubElement(elem, k) - # If v is a list, then it must be a list of tuples where the first - # item is the tag and the second item is the text value. - for v_tuple in v: - if len(v_tuple) == 2: - v_tag = v_tuple[0] - v_text = v_tuple[1] - # Don't include fields that are blank. - if v_text: - v_elem = XmlET.SubElement(sub_elem, v_tag) - v_elem.text = v_text - else: - sub_elem = XmlET.SubElement(elem, k) - sub_elem.text = v - return elem - return None - - -def parse_xml(file_name): - """Returns a parsed xml tree with comments intact.""" - error_message = '' - if not os.path.exists(file_name): - return None, "File does not exist %s" % str(file_name) - - with open(file_name, 'r') as fobj: - try: - tree = XmlET.parse(fobj, parser=XmlET.XMLParser(target=Py27CommentedTreeBuilder())) - except Exception as e: - error_message = "Exception attempting to parse %s: %s" % (str(file_name), str(e)) - log.exception(error_message) - return None, error_message - return tree, error_message diff --git a/lib/tool_shed/util/xml_util.py b/lib/tool_shed/util/xml_util.py new file mode 120000 index 00000000000..cb738288301 --- /dev/null +++ b/lib/tool_shed/util/xml_util.py @@ -0,0 +1 @@ +../../galaxy/util/tool_shed/xml_util.py \ No newline at end of file diff --git a/packages/app/galaxy/app.py b/packages/app/galaxy/app.py new file mode 120000 index 00000000000..61ec0a3dd4d --- /dev/null +++ b/packages/app/galaxy/app.py @@ -0,0 +1 @@ +../../../lib/galaxy/app.py \ No newline at end of file diff --git a/packages/app/galaxy/config.py b/packages/app/galaxy/config.py new file mode 120000 index 00000000000..7e61acb3d31 --- /dev/null +++ b/packages/app/galaxy/config.py @@ -0,0 +1 @@ +../../../lib/galaxy/config.py \ No newline at end of file diff --git a/packages/app/galaxy/config_watchers.py b/packages/app/galaxy/config_watchers.py new file mode 120000 index 00000000000..ffd3e6bd109 --- /dev/null +++ b/packages/app/galaxy/config_watchers.py @@ -0,0 +1 @@ +../../../lib/galaxy/config_watchers.py \ No newline at end of file diff --git a/packages/app/galaxy/main.py b/packages/app/galaxy/main.py new file mode 120000 index 00000000000..6b458fd3ff8 --- /dev/null +++ b/packages/app/galaxy/main.py @@ -0,0 +1 @@ +../../../lib/galaxy/main.py \ No newline at end of file diff --git a/packages/app/galaxy/queue_worker.py b/packages/app/galaxy/queue_worker.py new file mode 120000 index 00000000000..7fcb68fbea9 --- /dev/null +++ b/packages/app/galaxy/queue_worker.py @@ -0,0 +1 @@ +../../../lib/galaxy/queue_worker.py \ No newline at end of file diff --git a/packages/app/galaxy/queues.py b/packages/app/galaxy/queues.py new file mode 120000 index 00000000000..a5afab45290 --- /dev/null +++ b/packages/app/galaxy/queues.py @@ -0,0 +1 @@ +../../../lib/galaxy/queues.py \ No newline at end of file diff --git a/packages/app/galaxy/version.py b/packages/app/galaxy/version.py new file mode 120000 index 00000000000..42d972d9994 --- /dev/null +++ b/packages/app/galaxy/version.py @@ -0,0 +1 @@ +../../../lib/galaxy/version.py \ No newline at end of file diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 8cebfd65d71..40d2a9baed0 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -1,13 +1,16 @@ galaxy-objectstore -galaxy-util +galaxy-sequence-utils==1.1.3 +galaxy-util[template] bdbag bx-python h5py isa-rwval +parsley numpy<=1.16 pycryptodome pysam social_auth_core SQLAlchemy==1.2 sqlalchemy-migrate +sqlalchemy-utils WebOb diff --git a/packages/data/tests/unittest_utils/__init__.py b/packages/data/tests/unittest_utils/__init__.py index 2115ad35783..290bb9ee29b 120000 --- a/packages/data/tests/unittest_utils/__init__.py +++ b/packages/data/tests/unittest_utils/__init__.py @@ -1 +1 @@ -../../test/unit/unittest_utils/__init__.py \ No newline at end of file +../../../../test/unit/unittest_utils/__init__.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/tempfilecache.py b/packages/data/tests/unittest_utils/tempfilecache.py index 93fb83f1df7..588f6eec7d8 120000 --- a/packages/data/tests/unittest_utils/tempfilecache.py +++ b/packages/data/tests/unittest_utils/tempfilecache.py @@ -1 +1 @@ -../../test/unit/unittest_utils/tempfilecache.py \ No newline at end of file +../../../../test/unit/unittest_utils/tempfilecache.py \ No newline at end of file diff --git a/packages/data/tests/unittest_utils/utility.py b/packages/data/tests/unittest_utils/utility.py index 54dd4ca7a3b..3c69a8fd67e 120000 --- a/packages/data/tests/unittest_utils/utility.py +++ b/packages/data/tests/unittest_utils/utility.py @@ -1 +1 @@ -../../test/unit/unittest_utils/utility.py \ No newline at end of file +../../../../test/unit/unittest_utils/utility.py \ No newline at end of file diff --git a/packages/test.sh b/packages/test.sh index aaf70bd4464..43c4f70550c 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -31,8 +31,7 @@ PACKAGE_DIRS=( ) # containers has no tests, tool_util not yet working 100%, # data has many problems quota, tool shed install database, etc.. -RUN_TESTS=(1 1 1 0 1 0 1 0 1 0 0 0 0) - +RUN_TESTS=(1 1 1 0 1 1 1 0 1 0 0 0 0) for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do package_dir=${PACKAGE_DIRS[$i]} diff --git a/packages/util/requirements.txt b/packages/util/requirements.txt index e15e5db251e..aad3e2dc97a 100644 --- a/packages/util/requirements.txt +++ b/packages/util/requirements.txt @@ -5,4 +5,5 @@ docutils markupsafe packaging pyyaml +routes six>=1.9.0 diff --git a/run_tests.sh b/run_tests.sh index 2db31345c4f..884015751a9 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -11,7 +11,7 @@ cat <