mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-21 13:50:20 +08:00
Merge branch 'dev' of https://github.com/galaxyproject/galaxy into installable
This commit is contained in:
Nate Coraor
+1
-1
@@ -11,6 +11,7 @@ import galaxy.queues
|
||||
import galaxy.quota
|
||||
import galaxy.security
|
||||
from galaxy import config, job_metrics, jobs
|
||||
from galaxy.config_watchers import ConfigWatchers
|
||||
from galaxy.containers import build_container_interfaces
|
||||
from galaxy.managers.collections import DatasetCollectionManager
|
||||
from galaxy.managers.folders import FolderManager
|
||||
@@ -40,7 +41,6 @@ from galaxy.visualization.plugins.registry import VisualizationsRegistry
|
||||
from galaxy.web import url_for
|
||||
from galaxy.web.proxy import ProxyManager
|
||||
from galaxy.web_stack import application_stack_instance
|
||||
from galaxy.webapps.galaxy.config_watchers import ConfigWatchers
|
||||
from galaxy.webhooks import WebhooksRegistry
|
||||
from tool_shed.galaxy_install import (
|
||||
installed_repository_manager,
|
||||
|
||||
@@ -6,7 +6,7 @@ import sys
|
||||
|
||||
import bx.align.maf
|
||||
|
||||
from galaxy.tools.util import maf_utilities
|
||||
from galaxy.datatypes.util import maf_utilities
|
||||
|
||||
assert sys.version_info[:2] >= (2, 6)
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import sys
|
||||
|
||||
import bx.align.maf
|
||||
|
||||
from galaxy.tools.util import maf_utilities
|
||||
from galaxy.datatypes.util import maf_utilities
|
||||
|
||||
assert sys.version_info[:2] >= (2, 6)
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
1 2
|
||||
3 4
|
||||
@@ -0,0 +1,760 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
Provides wrappers and utilities for working with MAF files and alignments.
|
||||
"""
|
||||
# Dan Blankenberg
|
||||
from __future__ import print_function
|
||||
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import resource
|
||||
import sys
|
||||
import tempfile
|
||||
from copy import deepcopy
|
||||
from errno import EMFILE
|
||||
|
||||
import bx.align.maf
|
||||
import bx.interval_index_file
|
||||
import bx.intervals
|
||||
from six.moves import xrange
|
||||
|
||||
try:
|
||||
from string import maketrans
|
||||
except ImportError:
|
||||
maketrans = str.maketrans
|
||||
|
||||
assert sys.version_info[:2] >= (2, 6)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
GAP_CHARS = ['-']
|
||||
SRC_SPLIT_CHAR = '.'
|
||||
|
||||
|
||||
def src_split(src):
|
||||
fields = src.split(SRC_SPLIT_CHAR, 1)
|
||||
spec = fields.pop(0)
|
||||
if fields:
|
||||
chrom = fields.pop(0)
|
||||
else:
|
||||
chrom = spec
|
||||
return spec, chrom
|
||||
|
||||
|
||||
def src_merge(spec, chrom, contig=None):
|
||||
if None in [spec, chrom]:
|
||||
spec = chrom = spec or chrom
|
||||
return bx.align.maf.src_merge(spec, chrom, contig)
|
||||
|
||||
|
||||
def get_species_in_block(block):
|
||||
species = []
|
||||
for c in block.components:
|
||||
spec, chrom = src_split(c.src)
|
||||
if spec not in species:
|
||||
species.append(spec)
|
||||
return species
|
||||
|
||||
|
||||
def tool_fail(msg="Unknown Error"):
|
||||
print("Fatal Error: %s" % msg, file=sys.stderr)
|
||||
sys.exit()
|
||||
|
||||
|
||||
class TempFileHandler(object):
|
||||
'''
|
||||
Handles creating, opening, closing, and deleting of Temp files, with a
|
||||
maximum number of files open at one time.
|
||||
'''
|
||||
|
||||
DEFAULT_MAX_OPEN_FILES = max(resource.getrlimit(resource.RLIMIT_NOFILE)[0] / 2, 1)
|
||||
|
||||
def __init__(self, max_open_files=None, **kwds):
|
||||
if max_open_files is None:
|
||||
max_open_files = self.DEFAULT_MAX_OPEN_FILES
|
||||
self.max_open_files = max_open_files
|
||||
self.files = []
|
||||
self.open_file_indexes = []
|
||||
self.kwds = kwds
|
||||
|
||||
def get_open_tempfile(self, index=None, **kwds):
|
||||
if index is not None and index in self.open_file_indexes:
|
||||
self.open_file_indexes.remove(index)
|
||||
else:
|
||||
if self.max_open_files:
|
||||
while len(self.open_file_indexes) >= self.max_open_files:
|
||||
self.close(self.open_file_indexes[0])
|
||||
if index is None:
|
||||
index = len(self.files)
|
||||
temp_kwds = dict(self.kwds)
|
||||
temp_kwds.update(kwds)
|
||||
# Being able to use delete=True here, would simplify a bit,
|
||||
# but we support python2.4 in these tools
|
||||
while True:
|
||||
try:
|
||||
tmp_file = tempfile.NamedTemporaryFile(**temp_kwds)
|
||||
filename = tmp_file.name
|
||||
break
|
||||
except OSError as e:
|
||||
if self.open_file_indexes and e.errno == EMFILE:
|
||||
self.max_open_files = len(self.open_file_indexes)
|
||||
self.close(self.open_file_indexes[0])
|
||||
else:
|
||||
raise e
|
||||
tmp_file.close()
|
||||
self.files.append(open(filename, 'w+b'))
|
||||
else:
|
||||
while True:
|
||||
try:
|
||||
self.files[index] = open(self.files[index].name, 'r+b')
|
||||
break
|
||||
except OSError as e:
|
||||
if self.open_file_indexes and e.errno == EMFILE:
|
||||
self.max_open_files = len(self.open_file_indexes)
|
||||
self.close(self.open_file_indexes[0])
|
||||
else:
|
||||
raise e
|
||||
self.files[index].seek(0, 2)
|
||||
self.open_file_indexes.append(index)
|
||||
return index, self.files[index]
|
||||
|
||||
def close(self, index, delete=False):
|
||||
if index in self.open_file_indexes:
|
||||
self.open_file_indexes.remove(index)
|
||||
rval = self.files[index].close()
|
||||
if delete:
|
||||
try:
|
||||
os.unlink(self.files[index].name)
|
||||
except OSError:
|
||||
pass
|
||||
return rval
|
||||
|
||||
def flush(self, index):
|
||||
if index in self.open_file_indexes:
|
||||
self.files[index].flush()
|
||||
|
||||
def __del__(self):
|
||||
for i in xrange(len(self.files)):
|
||||
self.close(i, delete=True)
|
||||
|
||||
|
||||
# an object corresponding to a reference layered alignment
|
||||
class RegionAlignment(object):
|
||||
|
||||
DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca")
|
||||
MAX_SEQUENCE_SIZE = sys.maxsize # Maximum length of sequence allowed
|
||||
|
||||
def __init__(self, size, species=[], temp_file_handler=None):
|
||||
assert size <= self.MAX_SEQUENCE_SIZE, "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % (size, self.MAX_SEQUENCE_SIZE)
|
||||
self.size = size
|
||||
if not temp_file_handler:
|
||||
temp_file_handler = TempFileHandler()
|
||||
self.temp_file_handler = temp_file_handler
|
||||
self.sequences = {}
|
||||
if not isinstance(species, list):
|
||||
species = [species]
|
||||
for spec in species:
|
||||
self.add_species(spec)
|
||||
|
||||
# add a species to the alignment
|
||||
def add_species(self, species):
|
||||
# make temporary sequence files
|
||||
file_index, fh = self.temp_file_handler.get_open_tempfile()
|
||||
self.sequences[species] = file_index
|
||||
fh.write("-" * self.size)
|
||||
|
||||
# returns the names for species found in alignment, skipping names as requested
|
||||
def get_species_names(self, skip=[]):
|
||||
if not isinstance(skip, list):
|
||||
skip = [skip]
|
||||
names = list(self.sequences.keys())
|
||||
for name in skip:
|
||||
try:
|
||||
names.remove(name)
|
||||
except ValueError:
|
||||
pass
|
||||
return names
|
||||
|
||||
# returns the sequence for a species
|
||||
def get_sequence(self, species):
|
||||
file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species])
|
||||
fh.seek(0)
|
||||
return fh.read()
|
||||
|
||||
# returns the reverse complement of the sequence for a species
|
||||
def get_sequence_reverse_complement(self, species):
|
||||
complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)]
|
||||
complement.reverse()
|
||||
return "".join(complement)
|
||||
|
||||
# sets a position for a species
|
||||
def set_position(self, index, species, base):
|
||||
if len(base) != 1:
|
||||
raise Exception("A genomic position can only have a length of 1.")
|
||||
return self.set_range(index, species, base)
|
||||
# sets a range for a species
|
||||
|
||||
def set_range(self, index, species, bases):
|
||||
if index >= self.size or index < 0:
|
||||
raise Exception("Your index (%i) is out of range (0 - %i)." % (index, self.size - 1))
|
||||
if len(bases) == 0:
|
||||
raise Exception("A set of genomic positions can only have a positive length.")
|
||||
if species not in self.sequences.keys():
|
||||
self.add_species(species)
|
||||
file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species])
|
||||
fh.seek(index)
|
||||
fh.write(bases)
|
||||
|
||||
# Flush temp file of specified species, or all species
|
||||
def flush(self, species=None):
|
||||
if species is None:
|
||||
species = self.sequences.keys()
|
||||
elif not isinstance(species, list):
|
||||
species = [species]
|
||||
for spec in species:
|
||||
self.temp_file_handler.flush(self.sequences[spec])
|
||||
|
||||
|
||||
class GenomicRegionAlignment(RegionAlignment):
|
||||
|
||||
def __init__(self, start, end, species=[], temp_file_handler=None):
|
||||
RegionAlignment.__init__(self, end - start, species, temp_file_handler=temp_file_handler)
|
||||
self.start = start
|
||||
self.end = end
|
||||
|
||||
|
||||
class SplicedAlignment(object):
|
||||
|
||||
DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca")
|
||||
|
||||
def __init__(self, exon_starts, exon_ends, species=[], temp_file_handler=None):
|
||||
if not isinstance(exon_starts, list):
|
||||
exon_starts = [exon_starts]
|
||||
if not isinstance(exon_ends, list):
|
||||
exon_ends = [exon_ends]
|
||||
assert len(exon_starts) == len(exon_ends), "The number of starts does not match the number of sizes."
|
||||
self.exons = []
|
||||
if not temp_file_handler:
|
||||
temp_file_handler = TempFileHandler()
|
||||
self.temp_file_handler = temp_file_handler
|
||||
for i in range(len(exon_starts)):
|
||||
self.exons.append(GenomicRegionAlignment(exon_starts[i], exon_ends[i], species, temp_file_handler=temp_file_handler))
|
||||
|
||||
# returns the names for species found in alignment, skipping names as requested
|
||||
def get_species_names(self, skip=[]):
|
||||
if not isinstance(skip, list):
|
||||
skip = [skip]
|
||||
names = []
|
||||
for exon in self.exons:
|
||||
for name in exon.get_species_names(skip=skip):
|
||||
if name not in names:
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
# returns the sequence for a species
|
||||
def get_sequence(self, species):
|
||||
index, fh = self.temp_file_handler.get_open_tempfile()
|
||||
for exon in self.exons:
|
||||
if species in exon.get_species_names():
|
||||
seq = exon.get_sequence(species)
|
||||
# we need to refetch fh here, since exon.get_sequence( species ) uses a tempfile
|
||||
# and if max==1, it would close fh
|
||||
index, fh = self.temp_file_handler.get_open_tempfile(index)
|
||||
fh.write(seq)
|
||||
else:
|
||||
fh.write("-" * exon.size)
|
||||
fh.seek(0)
|
||||
rval = fh.read()
|
||||
self.temp_file_handler.close(index, delete=True)
|
||||
return rval
|
||||
|
||||
# returns the reverse complement of the sequence for a species
|
||||
def get_sequence_reverse_complement(self, species):
|
||||
complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)]
|
||||
complement.reverse()
|
||||
return "".join(complement)
|
||||
|
||||
# Start and end of coding region
|
||||
@property
|
||||
def start(self):
|
||||
return self.exons[0].start
|
||||
|
||||
@property
|
||||
def end(self):
|
||||
return self.exons[-1].end
|
||||
|
||||
|
||||
# Open a MAF index using a UID
|
||||
def maf_index_by_uid(maf_uid, index_location_file):
|
||||
for line in open(index_location_file):
|
||||
try:
|
||||
# read each line, if not enough fields, go to next line
|
||||
if line[0:1] == "#":
|
||||
continue
|
||||
fields = line.split('\t')
|
||||
if maf_uid == fields[1]:
|
||||
try:
|
||||
maf_files = fields[4].replace("\n", "").replace("\r", "").split(",")
|
||||
return bx.align.maf.MultiIndexed(maf_files, keep_open=True, parse_e_rows=False)
|
||||
except Exception as e:
|
||||
raise Exception('MAF UID (%s) found, but configuration appears to be malformed: %s' % (maf_uid, e))
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
# return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created
|
||||
def open_or_build_maf_index(maf_file, index_filename, species=None):
|
||||
try:
|
||||
return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), None)
|
||||
except Exception:
|
||||
return build_maf_index(maf_file, species=species)
|
||||
|
||||
|
||||
def build_maf_index_species_chromosomes(filename, index_species=None):
|
||||
species = []
|
||||
species_chromosomes = {}
|
||||
indexes = bx.interval_index_file.Indexes()
|
||||
blocks = 0
|
||||
try:
|
||||
maf_reader = bx.align.maf.Reader(open(filename))
|
||||
while True:
|
||||
pos = maf_reader.file.tell()
|
||||
block = next(maf_reader)
|
||||
if block is None:
|
||||
break
|
||||
blocks += 1
|
||||
for c in block.components:
|
||||
spec = c.src
|
||||
chrom = None
|
||||
if "." in spec:
|
||||
spec, chrom = spec.split(".", 1)
|
||||
if spec not in species:
|
||||
species.append(spec)
|
||||
species_chromosomes[spec] = []
|
||||
if chrom and chrom not in species_chromosomes[spec]:
|
||||
species_chromosomes[spec].append(chrom)
|
||||
if index_species is None or spec in index_species:
|
||||
forward_strand_start = c.forward_strand_start
|
||||
forward_strand_end = c.forward_strand_end
|
||||
try:
|
||||
forward_strand_start = int(forward_strand_start)
|
||||
forward_strand_end = int(forward_strand_end)
|
||||
except ValueError:
|
||||
continue # start and end are not integers, can't add component to index, goto next component
|
||||
# this likely only occurs when parse_e_rows is True?
|
||||
# could a species exist as only e rows? should the
|
||||
if forward_strand_end > forward_strand_start:
|
||||
# require positive length; i.e. certain lines have start = end = 0 and cannot be indexed
|
||||
indexes.add(c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size)
|
||||
except Exception as e:
|
||||
# most likely a bad MAF
|
||||
log.debug('Building MAF index on %s failed: %s' % (filename, e))
|
||||
return (None, [], {}, 0)
|
||||
return (indexes, species, species_chromosomes, blocks)
|
||||
|
||||
|
||||
# builds and returns ( index, index_filename ) for specified maf_file
|
||||
def build_maf_index(maf_file, species=None):
|
||||
indexes, found_species, species_chromosomes, blocks = build_maf_index_species_chromosomes(maf_file, species)
|
||||
if indexes is not None:
|
||||
fd, index_filename = tempfile.mkstemp()
|
||||
out = os.fdopen(fd, 'w')
|
||||
indexes.write(out)
|
||||
out.close()
|
||||
return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), index_filename)
|
||||
return (None, None)
|
||||
|
||||
|
||||
def component_overlaps_region(c, region):
|
||||
if c is None:
|
||||
return False
|
||||
start, end = c.get_forward_strand_start(), c.get_forward_strand_end()
|
||||
if region.start >= end or region.end <= start:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def chop_block_by_region(block, src, region, species=None, mincols=0):
|
||||
# This chopping method was designed to maintain consistency with how start/end padding gaps have been working in Galaxy thus far:
|
||||
# behavior as seen when forcing blocks to be '+' relative to src sequence (ref) and using block.slice_by_component( ref, slice_start, slice_end )
|
||||
# whether-or-not this is the 'correct' behavior is questionable, but this will at least maintain consistency
|
||||
# comments welcome
|
||||
slice_start = block.text_size # max for the min()
|
||||
slice_end = 0 # min for the max()
|
||||
old_score = block.score # save old score for later use
|
||||
# We no longer assume only one occurance of src per block, so we need to check them all
|
||||
for c in iter_components_by_src(block, src):
|
||||
if component_overlaps_region(c, region):
|
||||
if c.text is not None:
|
||||
rev_strand = False
|
||||
if c.strand == "-":
|
||||
# We want our coord_to_col coordinates to be returned from positive stranded component
|
||||
rev_strand = True
|
||||
c = c.reverse_complement()
|
||||
start = max(region.start, c.start)
|
||||
end = min(region.end, c.end)
|
||||
start = c.coord_to_col(start)
|
||||
end = c.coord_to_col(end)
|
||||
if rev_strand:
|
||||
# need to orient slice coordinates to the original block direction
|
||||
slice_len = end - start
|
||||
end = len(c.text) - start
|
||||
start = end - slice_len
|
||||
slice_start = min(start, slice_start)
|
||||
slice_end = max(end, slice_end)
|
||||
|
||||
if slice_start < slice_end:
|
||||
block = block.slice(slice_start, slice_end)
|
||||
if block.text_size > mincols:
|
||||
# restore old score, may not be accurate, but it is better than 0 for everything?
|
||||
block.score = old_score
|
||||
if species is not None:
|
||||
block = block.limit_to_species(species)
|
||||
block.remove_all_gap_columns()
|
||||
return block
|
||||
return None
|
||||
|
||||
|
||||
def orient_block_by_region(block, src, region, force_strand=None):
|
||||
# loop through components matching src,
|
||||
# make sure each of these components overlap region
|
||||
# cache strand for each of overlaping regions
|
||||
# if force_strand / region.strand not in strand cache, reverse complement
|
||||
# we could have 2 sequences with same src, overlapping region, on different strands, this would cause no reverse_complementing
|
||||
strands = [c.strand for c in iter_components_by_src(block, src) if component_overlaps_region(c, region)]
|
||||
if strands and (force_strand is None and region.strand not in strands) or (force_strand is not None and force_strand not in strands):
|
||||
block = block.reverse_complement()
|
||||
return block
|
||||
|
||||
|
||||
def get_oriented_chopped_blocks_for_region(index, src, region, species=None, mincols=0, force_strand=None):
|
||||
for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand):
|
||||
yield block
|
||||
|
||||
|
||||
def get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0, force_strand=None):
|
||||
for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols):
|
||||
yield orient_block_by_region(block, src, region, force_strand), idx, offset
|
||||
|
||||
|
||||
# split a block with multiple occurances of src into one block per src
|
||||
def iter_blocks_split_by_src(block, src):
|
||||
for src_c in iter_components_by_src(block, src):
|
||||
new_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes))
|
||||
new_block.text_size = block.text_size
|
||||
for c in block.components:
|
||||
if c == src_c or c.src != src:
|
||||
new_block.add_component(deepcopy(c)) # components have reference to alignment, don't want to lose reference to original alignment block in original components
|
||||
yield new_block
|
||||
|
||||
|
||||
# split a block into multiple blocks with all combinations of a species appearing only once per block
|
||||
def iter_blocks_split_by_species(block, species=None):
|
||||
def __split_components_by_species(components_by_species, new_block):
|
||||
if components_by_species:
|
||||
# more species with components to add to this block
|
||||
components_by_species = deepcopy(components_by_species)
|
||||
spec_comps = components_by_species.pop(0)
|
||||
for c in spec_comps:
|
||||
newer_block = deepcopy(new_block)
|
||||
newer_block.add_component(deepcopy(c))
|
||||
for value in __split_components_by_species(components_by_species, newer_block):
|
||||
yield value
|
||||
else:
|
||||
# no more components to add, yield this block
|
||||
yield new_block
|
||||
|
||||
# divide components by species
|
||||
spec_dict = {}
|
||||
if not species:
|
||||
species = []
|
||||
for c in block.components:
|
||||
spec, chrom = src_split(c.src)
|
||||
if spec not in spec_dict:
|
||||
spec_dict[spec] = []
|
||||
species.append(spec)
|
||||
spec_dict[spec].append(c)
|
||||
else:
|
||||
for spec in species:
|
||||
spec_dict[spec] = []
|
||||
for c in iter_components_by_src_start(block, spec):
|
||||
spec_dict[spec].append(c)
|
||||
|
||||
empty_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) # should we copy attributes?
|
||||
empty_block.text_size = block.text_size
|
||||
# call recursive function to split into each combo of spec/blocks
|
||||
for value in __split_components_by_species(list(spec_dict.values()), empty_block):
|
||||
sort_block_components_by_block(value, block) # restore original component order
|
||||
yield value
|
||||
|
||||
|
||||
# generator yielding only chopped and valid blocks for a specified region
|
||||
def get_chopped_blocks_for_region(index, src, region, species=None, mincols=0):
|
||||
for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols):
|
||||
yield block
|
||||
|
||||
|
||||
def get_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0):
|
||||
for block, idx, offset in index.get_as_iterator_with_index_and_offset(src, region.start, region.end):
|
||||
block = chop_block_by_region(block, src, region, species, mincols)
|
||||
if block is not None:
|
||||
yield block, idx, offset
|
||||
|
||||
|
||||
# returns a filled region alignment for specified regions
|
||||
def get_region_alignment(index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None):
|
||||
if species is not None:
|
||||
alignment = RegionAlignment(end - start, species, temp_file_handler=temp_file_handler)
|
||||
else:
|
||||
alignment = RegionAlignment(end - start, primary_species, temp_file_handler=temp_file_handler)
|
||||
return fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps)
|
||||
|
||||
|
||||
# reduces a block to only positions exisiting in the src provided
|
||||
def reduce_block_by_primary_genome(block, species, chromosome, region_start):
|
||||
# returns ( startIndex, {species:texts}
|
||||
# where texts' contents are reduced to only positions existing in the primary genome
|
||||
src = "%s.%s" % (species, chromosome)
|
||||
ref = block.get_component_by_src(src)
|
||||
start_offset = ref.start - region_start
|
||||
species_texts = {}
|
||||
for c in block.components:
|
||||
species_texts[c.src.split('.')[0]] = list(c.text)
|
||||
# remove locations which are gaps in the primary species, starting from the downstream end
|
||||
for i in range(len(species_texts[species]) - 1, -1, -1):
|
||||
if species_texts[species][i] == '-':
|
||||
for text in species_texts.values():
|
||||
text.pop(i)
|
||||
for spec, text in species_texts.items():
|
||||
species_texts[spec] = ''.join(text)
|
||||
return (start_offset, species_texts)
|
||||
|
||||
|
||||
# fills a region alignment
|
||||
def fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True):
|
||||
region = bx.intervals.Interval(start, end)
|
||||
region.chrom = chrom
|
||||
region.strand = strand
|
||||
primary_src = "%s.%s" % (primary_species, chrom)
|
||||
|
||||
# Order blocks overlaping this position by score, lowest first
|
||||
blocks = []
|
||||
for block, idx, offset in index.get_as_iterator_with_index_and_offset(primary_src, start, end):
|
||||
score = float(block.score)
|
||||
for i in range(0, len(blocks)):
|
||||
if score < blocks[i][0]:
|
||||
blocks.insert(i, (score, idx, offset))
|
||||
break
|
||||
else:
|
||||
blocks.append((score, idx, offset))
|
||||
|
||||
# gap_chars_tuple = tuple( GAP_CHARS )
|
||||
gap_chars_str = ''.join(GAP_CHARS)
|
||||
# Loop through ordered blocks and layer by increasing score
|
||||
for block_dict in blocks:
|
||||
for block in iter_blocks_split_by_species(block_dict[1].get_at_offset(block_dict[2])): # need to handle each occurance of sequence in block seperately
|
||||
if component_overlaps_region(block.get_component_by_src(primary_src), region):
|
||||
block = chop_block_by_region(block, primary_src, region, species, mincols) # chop block
|
||||
block = orient_block_by_region(block, primary_src, region) # orient block
|
||||
start_offset, species_texts = reduce_block_by_primary_genome(block, primary_species, chrom, start)
|
||||
for spec, text in species_texts.items():
|
||||
# we should trim gaps from both sides, since these are not positions in this species genome (sequence)
|
||||
text = text.rstrip(gap_chars_str)
|
||||
gap_offset = 0
|
||||
# while text.startswith( gap_chars_tuple ):
|
||||
while True in [text.startswith(gap_char) for gap_char in GAP_CHARS]: # python2.4 doesn't accept a tuple for .startswith()
|
||||
gap_offset += 1
|
||||
text = text[1:]
|
||||
if not text:
|
||||
break
|
||||
if text:
|
||||
if overwrite_with_gaps:
|
||||
alignment.set_range(start_offset + gap_offset, spec, text)
|
||||
else:
|
||||
for i, char in enumerate(text):
|
||||
if char not in GAP_CHARS:
|
||||
alignment.set_position(start_offset + gap_offset + i, spec, char)
|
||||
return alignment
|
||||
|
||||
|
||||
# returns a filled spliced region alignment for specified region with start and end lists
|
||||
def get_spliced_region_alignment(index, primary_species, chrom, starts, ends, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None):
|
||||
# create spliced alignment object
|
||||
if species is not None:
|
||||
alignment = SplicedAlignment(starts, ends, species, temp_file_handler=temp_file_handler)
|
||||
else:
|
||||
alignment = SplicedAlignment(starts, ends, [primary_species], temp_file_handler=temp_file_handler)
|
||||
for exon in alignment.exons:
|
||||
fill_region_alignment(exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps)
|
||||
return alignment
|
||||
|
||||
|
||||
# loop through string array, only return non-commented lines
|
||||
def line_enumerator(lines, comment_start='#'):
|
||||
i = 0
|
||||
for line in lines:
|
||||
if not line.startswith(comment_start):
|
||||
i += 1
|
||||
yield (i, line)
|
||||
|
||||
|
||||
# read a GeneBed file, return list of starts, ends, raw fields
|
||||
def get_starts_ends_fields_from_gene_bed(line):
|
||||
# Starts and ends for exons
|
||||
starts = []
|
||||
ends = []
|
||||
|
||||
fields = line.split()
|
||||
# Requires atleast 12 BED columns
|
||||
if len(fields) < 12:
|
||||
raise Exception("Not a proper 12 column BED line (%s)." % line)
|
||||
tx_start = int(fields[1])
|
||||
strand = fields[5]
|
||||
if strand != '-':
|
||||
strand = '+' # Default strand is +
|
||||
cds_start = int(fields[6])
|
||||
cds_end = int(fields[7])
|
||||
|
||||
# Calculate and store starts and ends of coding exons
|
||||
region_start, region_end = cds_start, cds_end
|
||||
exon_starts = list(map(int, fields[11].rstrip(',\n').split(',')))
|
||||
exon_starts = [x + tx_start for x in exon_starts]
|
||||
exon_ends = list(map(int, fields[10].rstrip(',').split(',')))
|
||||
exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)]
|
||||
for start, end in zip(exon_starts, exon_ends):
|
||||
start = max(start, region_start)
|
||||
end = min(end, region_end)
|
||||
if start < end:
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
return (starts, ends, fields)
|
||||
|
||||
|
||||
def iter_components_by_src(block, src):
|
||||
for c in block.components:
|
||||
if c.src == src:
|
||||
yield c
|
||||
|
||||
|
||||
def get_components_by_src(block, src):
|
||||
return [value for value in iter_components_by_src(block, src)]
|
||||
|
||||
|
||||
def iter_components_by_src_start(block, src):
|
||||
for c in block.components:
|
||||
if c.src.startswith(src):
|
||||
yield c
|
||||
|
||||
|
||||
def get_components_by_src_start(block, src):
|
||||
return [value for value in iter_components_by_src_start(block, src)]
|
||||
|
||||
|
||||
def sort_block_components_by_block(block1, block2):
|
||||
# orders the components in block1 by the index of the component in block2
|
||||
# block1 must be a subset of block2
|
||||
# occurs in-place
|
||||
return block1.components.sort(key=functools.cmp_to_key(lambda x, y: block2.components.index(x) - block2.components.index(y)))
|
||||
|
||||
|
||||
def get_species_in_maf(maf_filename):
|
||||
species = []
|
||||
for block in bx.align.maf.Reader(open(maf_filename)):
|
||||
for spec in get_species_in_block(block):
|
||||
if spec not in species:
|
||||
species.append(spec)
|
||||
return species
|
||||
|
||||
|
||||
def parse_species_option(species):
|
||||
if species:
|
||||
species = species.split(',')
|
||||
if 'None' not in species:
|
||||
return species
|
||||
return None # provided species was '', None, or had 'None' in it
|
||||
|
||||
|
||||
def remove_temp_index_file(index_filename):
|
||||
try:
|
||||
os.unlink(index_filename)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Below are methods to deal with FASTA files
|
||||
|
||||
|
||||
def get_fasta_header(component, attributes={}, suffix=None):
|
||||
header = ">%s(%s):%i-%i|" % (component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end())
|
||||
for key, value in attributes.items():
|
||||
header = "%s%s=%s|" % (header, key, value)
|
||||
if suffix:
|
||||
header = "%s%s" % (header, suffix)
|
||||
else:
|
||||
header = "%s%s" % (header, src_split(component.src)[0])
|
||||
return header
|
||||
|
||||
|
||||
def get_attributes_from_fasta_header(header):
|
||||
if not header:
|
||||
return {}
|
||||
attributes = {}
|
||||
header = header.lstrip('>')
|
||||
header = header.strip()
|
||||
fields = header.split('|')
|
||||
try:
|
||||
region = fields[0]
|
||||
region = region.split('(', 1)
|
||||
temp = region[0].split('.', 1)
|
||||
attributes['species'] = temp[0]
|
||||
if len(temp) == 2:
|
||||
attributes['chrom'] = temp[1]
|
||||
else:
|
||||
attributes['chrom'] = temp[0]
|
||||
region = region[1].split(')', 1)
|
||||
attributes['strand'] = region[0]
|
||||
region = region[1].lstrip(':').split('-')
|
||||
attributes['start'] = int(region[0])
|
||||
attributes['end'] = int(region[1])
|
||||
except Exception:
|
||||
# fields 0 is not a region coordinate
|
||||
pass
|
||||
if len(fields) > 2:
|
||||
for i in range(1, len(fields) - 1):
|
||||
prop = fields[i].split('=', 1)
|
||||
if len(prop) == 2:
|
||||
attributes[prop[0]] = prop[1]
|
||||
if len(fields) > 1:
|
||||
attributes['__suffix__'] = fields[-1]
|
||||
return attributes
|
||||
|
||||
|
||||
def iter_fasta_alignment(filename):
|
||||
class fastaComponent(object):
|
||||
def __init__(self, species, text=""):
|
||||
self.species = species
|
||||
self.text = text
|
||||
|
||||
def extend(self, text):
|
||||
self.text = self.text + text.replace('\n', '').replace('\r', '').strip()
|
||||
# yields a list of fastaComponents for a FASTA file
|
||||
f = open(filename, 'rb')
|
||||
components = []
|
||||
# cur_component = None
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
if components:
|
||||
yield components
|
||||
return
|
||||
line = line.strip()
|
||||
if not line:
|
||||
if components:
|
||||
yield components
|
||||
components = []
|
||||
elif line.startswith('>'):
|
||||
attributes = get_attributes_from_fasta_header(line)
|
||||
components.append(fastaComponent(attributes['species']))
|
||||
elif components:
|
||||
components[-1].extend(line)
|
||||
@@ -4,7 +4,7 @@ import os
|
||||
from galaxy.util import asbool
|
||||
from galaxy.util.bunch import Bunch
|
||||
from galaxy.util.dictifiable import Dictifiable
|
||||
from tool_shed.util import common_util
|
||||
from galaxy.util.tool_shed import common_util
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -1,760 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
Provides wrappers and utilities for working with MAF files and alignments.
|
||||
"""
|
||||
# Dan Blankenberg
|
||||
from __future__ import print_function
|
||||
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import resource
|
||||
import sys
|
||||
import tempfile
|
||||
from copy import deepcopy
|
||||
from errno import EMFILE
|
||||
|
||||
import bx.align.maf
|
||||
import bx.interval_index_file
|
||||
import bx.intervals
|
||||
from six.moves import xrange
|
||||
|
||||
try:
|
||||
from string import maketrans
|
||||
except ImportError:
|
||||
maketrans = str.maketrans
|
||||
|
||||
assert sys.version_info[:2] >= (2, 6)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
GAP_CHARS = ['-']
|
||||
SRC_SPLIT_CHAR = '.'
|
||||
|
||||
|
||||
def src_split(src):
|
||||
fields = src.split(SRC_SPLIT_CHAR, 1)
|
||||
spec = fields.pop(0)
|
||||
if fields:
|
||||
chrom = fields.pop(0)
|
||||
else:
|
||||
chrom = spec
|
||||
return spec, chrom
|
||||
|
||||
|
||||
def src_merge(spec, chrom, contig=None):
|
||||
if None in [spec, chrom]:
|
||||
spec = chrom = spec or chrom
|
||||
return bx.align.maf.src_merge(spec, chrom, contig)
|
||||
|
||||
|
||||
def get_species_in_block(block):
|
||||
species = []
|
||||
for c in block.components:
|
||||
spec, chrom = src_split(c.src)
|
||||
if spec not in species:
|
||||
species.append(spec)
|
||||
return species
|
||||
|
||||
|
||||
def tool_fail(msg="Unknown Error"):
|
||||
print("Fatal Error: %s" % msg, file=sys.stderr)
|
||||
sys.exit()
|
||||
|
||||
|
||||
class TempFileHandler(object):
|
||||
'''
|
||||
Handles creating, opening, closing, and deleting of Temp files, with a
|
||||
maximum number of files open at one time.
|
||||
'''
|
||||
|
||||
DEFAULT_MAX_OPEN_FILES = max(resource.getrlimit(resource.RLIMIT_NOFILE)[0] / 2, 1)
|
||||
|
||||
def __init__(self, max_open_files=None, **kwds):
|
||||
if max_open_files is None:
|
||||
max_open_files = self.DEFAULT_MAX_OPEN_FILES
|
||||
self.max_open_files = max_open_files
|
||||
self.files = []
|
||||
self.open_file_indexes = []
|
||||
self.kwds = kwds
|
||||
|
||||
def get_open_tempfile(self, index=None, **kwds):
|
||||
if index is not None and index in self.open_file_indexes:
|
||||
self.open_file_indexes.remove(index)
|
||||
else:
|
||||
if self.max_open_files:
|
||||
while len(self.open_file_indexes) >= self.max_open_files:
|
||||
self.close(self.open_file_indexes[0])
|
||||
if index is None:
|
||||
index = len(self.files)
|
||||
temp_kwds = dict(self.kwds)
|
||||
temp_kwds.update(kwds)
|
||||
# Being able to use delete=True here, would simplify a bit,
|
||||
# but we support python2.4 in these tools
|
||||
while True:
|
||||
try:
|
||||
tmp_file = tempfile.NamedTemporaryFile(**temp_kwds)
|
||||
filename = tmp_file.name
|
||||
break
|
||||
except OSError as e:
|
||||
if self.open_file_indexes and e.errno == EMFILE:
|
||||
self.max_open_files = len(self.open_file_indexes)
|
||||
self.close(self.open_file_indexes[0])
|
||||
else:
|
||||
raise e
|
||||
tmp_file.close()
|
||||
self.files.append(open(filename, 'w+b'))
|
||||
else:
|
||||
while True:
|
||||
try:
|
||||
self.files[index] = open(self.files[index].name, 'r+b')
|
||||
break
|
||||
except OSError as e:
|
||||
if self.open_file_indexes and e.errno == EMFILE:
|
||||
self.max_open_files = len(self.open_file_indexes)
|
||||
self.close(self.open_file_indexes[0])
|
||||
else:
|
||||
raise e
|
||||
self.files[index].seek(0, 2)
|
||||
self.open_file_indexes.append(index)
|
||||
return index, self.files[index]
|
||||
|
||||
def close(self, index, delete=False):
|
||||
if index in self.open_file_indexes:
|
||||
self.open_file_indexes.remove(index)
|
||||
rval = self.files[index].close()
|
||||
if delete:
|
||||
try:
|
||||
os.unlink(self.files[index].name)
|
||||
except OSError:
|
||||
pass
|
||||
return rval
|
||||
|
||||
def flush(self, index):
|
||||
if index in self.open_file_indexes:
|
||||
self.files[index].flush()
|
||||
|
||||
def __del__(self):
|
||||
for i in xrange(len(self.files)):
|
||||
self.close(i, delete=True)
|
||||
|
||||
|
||||
# an object corresponding to a reference layered alignment
|
||||
class RegionAlignment(object):
|
||||
|
||||
DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca")
|
||||
MAX_SEQUENCE_SIZE = sys.maxsize # Maximum length of sequence allowed
|
||||
|
||||
def __init__(self, size, species=[], temp_file_handler=None):
|
||||
assert size <= self.MAX_SEQUENCE_SIZE, "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % (size, self.MAX_SEQUENCE_SIZE)
|
||||
self.size = size
|
||||
if not temp_file_handler:
|
||||
temp_file_handler = TempFileHandler()
|
||||
self.temp_file_handler = temp_file_handler
|
||||
self.sequences = {}
|
||||
if not isinstance(species, list):
|
||||
species = [species]
|
||||
for spec in species:
|
||||
self.add_species(spec)
|
||||
|
||||
# add a species to the alignment
|
||||
def add_species(self, species):
|
||||
# make temporary sequence files
|
||||
file_index, fh = self.temp_file_handler.get_open_tempfile()
|
||||
self.sequences[species] = file_index
|
||||
fh.write("-" * self.size)
|
||||
|
||||
# returns the names for species found in alignment, skipping names as requested
|
||||
def get_species_names(self, skip=[]):
|
||||
if not isinstance(skip, list):
|
||||
skip = [skip]
|
||||
names = list(self.sequences.keys())
|
||||
for name in skip:
|
||||
try:
|
||||
names.remove(name)
|
||||
except ValueError:
|
||||
pass
|
||||
return names
|
||||
|
||||
# returns the sequence for a species
|
||||
def get_sequence(self, species):
|
||||
file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species])
|
||||
fh.seek(0)
|
||||
return fh.read()
|
||||
|
||||
# returns the reverse complement of the sequence for a species
|
||||
def get_sequence_reverse_complement(self, species):
|
||||
complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)]
|
||||
complement.reverse()
|
||||
return "".join(complement)
|
||||
|
||||
# sets a position for a species
|
||||
def set_position(self, index, species, base):
|
||||
if len(base) != 1:
|
||||
raise Exception("A genomic position can only have a length of 1.")
|
||||
return self.set_range(index, species, base)
|
||||
# sets a range for a species
|
||||
|
||||
def set_range(self, index, species, bases):
|
||||
if index >= self.size or index < 0:
|
||||
raise Exception("Your index (%i) is out of range (0 - %i)." % (index, self.size - 1))
|
||||
if len(bases) == 0:
|
||||
raise Exception("A set of genomic positions can only have a positive length.")
|
||||
if species not in self.sequences.keys():
|
||||
self.add_species(species)
|
||||
file_index, fh = self.temp_file_handler.get_open_tempfile(self.sequences[species])
|
||||
fh.seek(index)
|
||||
fh.write(bases)
|
||||
|
||||
# Flush temp file of specified species, or all species
|
||||
def flush(self, species=None):
|
||||
if species is None:
|
||||
species = self.sequences.keys()
|
||||
elif not isinstance(species, list):
|
||||
species = [species]
|
||||
for spec in species:
|
||||
self.temp_file_handler.flush(self.sequences[spec])
|
||||
|
||||
|
||||
class GenomicRegionAlignment(RegionAlignment):
|
||||
|
||||
def __init__(self, start, end, species=[], temp_file_handler=None):
|
||||
RegionAlignment.__init__(self, end - start, species, temp_file_handler=temp_file_handler)
|
||||
self.start = start
|
||||
self.end = end
|
||||
|
||||
|
||||
class SplicedAlignment(object):
|
||||
|
||||
DNA_COMPLEMENT = maketrans("ACGTacgt", "TGCAtgca")
|
||||
|
||||
def __init__(self, exon_starts, exon_ends, species=[], temp_file_handler=None):
|
||||
if not isinstance(exon_starts, list):
|
||||
exon_starts = [exon_starts]
|
||||
if not isinstance(exon_ends, list):
|
||||
exon_ends = [exon_ends]
|
||||
assert len(exon_starts) == len(exon_ends), "The number of starts does not match the number of sizes."
|
||||
self.exons = []
|
||||
if not temp_file_handler:
|
||||
temp_file_handler = TempFileHandler()
|
||||
self.temp_file_handler = temp_file_handler
|
||||
for i in range(len(exon_starts)):
|
||||
self.exons.append(GenomicRegionAlignment(exon_starts[i], exon_ends[i], species, temp_file_handler=temp_file_handler))
|
||||
|
||||
# returns the names for species found in alignment, skipping names as requested
|
||||
def get_species_names(self, skip=[]):
|
||||
if not isinstance(skip, list):
|
||||
skip = [skip]
|
||||
names = []
|
||||
for exon in self.exons:
|
||||
for name in exon.get_species_names(skip=skip):
|
||||
if name not in names:
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
# returns the sequence for a species
|
||||
def get_sequence(self, species):
|
||||
index, fh = self.temp_file_handler.get_open_tempfile()
|
||||
for exon in self.exons:
|
||||
if species in exon.get_species_names():
|
||||
seq = exon.get_sequence(species)
|
||||
# we need to refetch fh here, since exon.get_sequence( species ) uses a tempfile
|
||||
# and if max==1, it would close fh
|
||||
index, fh = self.temp_file_handler.get_open_tempfile(index)
|
||||
fh.write(seq)
|
||||
else:
|
||||
fh.write("-" * exon.size)
|
||||
fh.seek(0)
|
||||
rval = fh.read()
|
||||
self.temp_file_handler.close(index, delete=True)
|
||||
return rval
|
||||
|
||||
# returns the reverse complement of the sequence for a species
|
||||
def get_sequence_reverse_complement(self, species):
|
||||
complement = [base for base in self.get_sequence(species).translate(self.DNA_COMPLEMENT)]
|
||||
complement.reverse()
|
||||
return "".join(complement)
|
||||
|
||||
# Start and end of coding region
|
||||
@property
|
||||
def start(self):
|
||||
return self.exons[0].start
|
||||
|
||||
@property
|
||||
def end(self):
|
||||
return self.exons[-1].end
|
||||
|
||||
|
||||
# Open a MAF index using a UID
|
||||
def maf_index_by_uid(maf_uid, index_location_file):
|
||||
for line in open(index_location_file):
|
||||
try:
|
||||
# read each line, if not enough fields, go to next line
|
||||
if line[0:1] == "#":
|
||||
continue
|
||||
fields = line.split('\t')
|
||||
if maf_uid == fields[1]:
|
||||
try:
|
||||
maf_files = fields[4].replace("\n", "").replace("\r", "").split(",")
|
||||
return bx.align.maf.MultiIndexed(maf_files, keep_open=True, parse_e_rows=False)
|
||||
except Exception as e:
|
||||
raise Exception('MAF UID (%s) found, but configuration appears to be malformed: %s' % (maf_uid, e))
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
# return ( index, temp_index_filename ) for user maf, if available, or build one and return it, return None when no tempfile is created
|
||||
def open_or_build_maf_index(maf_file, index_filename, species=None):
|
||||
try:
|
||||
return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), None)
|
||||
except Exception:
|
||||
return build_maf_index(maf_file, species=species)
|
||||
|
||||
|
||||
def build_maf_index_species_chromosomes(filename, index_species=None):
|
||||
species = []
|
||||
species_chromosomes = {}
|
||||
indexes = bx.interval_index_file.Indexes()
|
||||
blocks = 0
|
||||
try:
|
||||
maf_reader = bx.align.maf.Reader(open(filename))
|
||||
while True:
|
||||
pos = maf_reader.file.tell()
|
||||
block = next(maf_reader)
|
||||
if block is None:
|
||||
break
|
||||
blocks += 1
|
||||
for c in block.components:
|
||||
spec = c.src
|
||||
chrom = None
|
||||
if "." in spec:
|
||||
spec, chrom = spec.split(".", 1)
|
||||
if spec not in species:
|
||||
species.append(spec)
|
||||
species_chromosomes[spec] = []
|
||||
if chrom and chrom not in species_chromosomes[spec]:
|
||||
species_chromosomes[spec].append(chrom)
|
||||
if index_species is None or spec in index_species:
|
||||
forward_strand_start = c.forward_strand_start
|
||||
forward_strand_end = c.forward_strand_end
|
||||
try:
|
||||
forward_strand_start = int(forward_strand_start)
|
||||
forward_strand_end = int(forward_strand_end)
|
||||
except ValueError:
|
||||
continue # start and end are not integers, can't add component to index, goto next component
|
||||
# this likely only occurs when parse_e_rows is True?
|
||||
# could a species exist as only e rows? should the
|
||||
if forward_strand_end > forward_strand_start:
|
||||
# require positive length; i.e. certain lines have start = end = 0 and cannot be indexed
|
||||
indexes.add(c.src, forward_strand_start, forward_strand_end, pos, max=c.src_size)
|
||||
except Exception as e:
|
||||
# most likely a bad MAF
|
||||
log.debug('Building MAF index on %s failed: %s' % (filename, e))
|
||||
return (None, [], {}, 0)
|
||||
return (indexes, species, species_chromosomes, blocks)
|
||||
|
||||
|
||||
# builds and returns ( index, index_filename ) for specified maf_file
|
||||
def build_maf_index(maf_file, species=None):
|
||||
indexes, found_species, species_chromosomes, blocks = build_maf_index_species_chromosomes(maf_file, species)
|
||||
if indexes is not None:
|
||||
fd, index_filename = tempfile.mkstemp()
|
||||
out = os.fdopen(fd, 'w')
|
||||
indexes.write(out)
|
||||
out.close()
|
||||
return (bx.align.maf.Indexed(maf_file, index_filename=index_filename, keep_open=True, parse_e_rows=False), index_filename)
|
||||
return (None, None)
|
||||
|
||||
|
||||
def component_overlaps_region(c, region):
|
||||
if c is None:
|
||||
return False
|
||||
start, end = c.get_forward_strand_start(), c.get_forward_strand_end()
|
||||
if region.start >= end or region.end <= start:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def chop_block_by_region(block, src, region, species=None, mincols=0):
|
||||
# This chopping method was designed to maintain consistency with how start/end padding gaps have been working in Galaxy thus far:
|
||||
# behavior as seen when forcing blocks to be '+' relative to src sequence (ref) and using block.slice_by_component( ref, slice_start, slice_end )
|
||||
# whether-or-not this is the 'correct' behavior is questionable, but this will at least maintain consistency
|
||||
# comments welcome
|
||||
slice_start = block.text_size # max for the min()
|
||||
slice_end = 0 # min for the max()
|
||||
old_score = block.score # save old score for later use
|
||||
# We no longer assume only one occurance of src per block, so we need to check them all
|
||||
for c in iter_components_by_src(block, src):
|
||||
if component_overlaps_region(c, region):
|
||||
if c.text is not None:
|
||||
rev_strand = False
|
||||
if c.strand == "-":
|
||||
# We want our coord_to_col coordinates to be returned from positive stranded component
|
||||
rev_strand = True
|
||||
c = c.reverse_complement()
|
||||
start = max(region.start, c.start)
|
||||
end = min(region.end, c.end)
|
||||
start = c.coord_to_col(start)
|
||||
end = c.coord_to_col(end)
|
||||
if rev_strand:
|
||||
# need to orient slice coordinates to the original block direction
|
||||
slice_len = end - start
|
||||
end = len(c.text) - start
|
||||
start = end - slice_len
|
||||
slice_start = min(start, slice_start)
|
||||
slice_end = max(end, slice_end)
|
||||
|
||||
if slice_start < slice_end:
|
||||
block = block.slice(slice_start, slice_end)
|
||||
if block.text_size > mincols:
|
||||
# restore old score, may not be accurate, but it is better than 0 for everything?
|
||||
block.score = old_score
|
||||
if species is not None:
|
||||
block = block.limit_to_species(species)
|
||||
block.remove_all_gap_columns()
|
||||
return block
|
||||
return None
|
||||
|
||||
|
||||
def orient_block_by_region(block, src, region, force_strand=None):
|
||||
# loop through components matching src,
|
||||
# make sure each of these components overlap region
|
||||
# cache strand for each of overlaping regions
|
||||
# if force_strand / region.strand not in strand cache, reverse complement
|
||||
# we could have 2 sequences with same src, overlapping region, on different strands, this would cause no reverse_complementing
|
||||
strands = [c.strand for c in iter_components_by_src(block, src) if component_overlaps_region(c, region)]
|
||||
if strands and (force_strand is None and region.strand not in strands) or (force_strand is not None and force_strand not in strands):
|
||||
block = block.reverse_complement()
|
||||
return block
|
||||
|
||||
|
||||
def get_oriented_chopped_blocks_for_region(index, src, region, species=None, mincols=0, force_strand=None):
|
||||
for block, idx, offset in get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols, force_strand):
|
||||
yield block
|
||||
|
||||
|
||||
def get_oriented_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0, force_strand=None):
|
||||
for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols):
|
||||
yield orient_block_by_region(block, src, region, force_strand), idx, offset
|
||||
|
||||
|
||||
# split a block with multiple occurances of src into one block per src
|
||||
def iter_blocks_split_by_src(block, src):
|
||||
for src_c in iter_components_by_src(block, src):
|
||||
new_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes))
|
||||
new_block.text_size = block.text_size
|
||||
for c in block.components:
|
||||
if c == src_c or c.src != src:
|
||||
new_block.add_component(deepcopy(c)) # components have reference to alignment, don't want to lose reference to original alignment block in original components
|
||||
yield new_block
|
||||
|
||||
|
||||
# split a block into multiple blocks with all combinations of a species appearing only once per block
|
||||
def iter_blocks_split_by_species(block, species=None):
|
||||
def __split_components_by_species(components_by_species, new_block):
|
||||
if components_by_species:
|
||||
# more species with components to add to this block
|
||||
components_by_species = deepcopy(components_by_species)
|
||||
spec_comps = components_by_species.pop(0)
|
||||
for c in spec_comps:
|
||||
newer_block = deepcopy(new_block)
|
||||
newer_block.add_component(deepcopy(c))
|
||||
for value in __split_components_by_species(components_by_species, newer_block):
|
||||
yield value
|
||||
else:
|
||||
# no more components to add, yield this block
|
||||
yield new_block
|
||||
|
||||
# divide components by species
|
||||
spec_dict = {}
|
||||
if not species:
|
||||
species = []
|
||||
for c in block.components:
|
||||
spec, chrom = src_split(c.src)
|
||||
if spec not in spec_dict:
|
||||
spec_dict[spec] = []
|
||||
species.append(spec)
|
||||
spec_dict[spec].append(c)
|
||||
else:
|
||||
for spec in species:
|
||||
spec_dict[spec] = []
|
||||
for c in iter_components_by_src_start(block, spec):
|
||||
spec_dict[spec].append(c)
|
||||
|
||||
empty_block = bx.align.Alignment(score=block.score, attributes=deepcopy(block.attributes)) # should we copy attributes?
|
||||
empty_block.text_size = block.text_size
|
||||
# call recursive function to split into each combo of spec/blocks
|
||||
for value in __split_components_by_species(list(spec_dict.values()), empty_block):
|
||||
sort_block_components_by_block(value, block) # restore original component order
|
||||
yield value
|
||||
|
||||
|
||||
# generator yielding only chopped and valid blocks for a specified region
|
||||
def get_chopped_blocks_for_region(index, src, region, species=None, mincols=0):
|
||||
for block, idx, offset in get_chopped_blocks_with_index_offset_for_region(index, src, region, species, mincols):
|
||||
yield block
|
||||
|
||||
|
||||
def get_chopped_blocks_with_index_offset_for_region(index, src, region, species=None, mincols=0):
|
||||
for block, idx, offset in index.get_as_iterator_with_index_and_offset(src, region.start, region.end):
|
||||
block = chop_block_by_region(block, src, region, species, mincols)
|
||||
if block is not None:
|
||||
yield block, idx, offset
|
||||
|
||||
|
||||
# returns a filled region alignment for specified regions
|
||||
def get_region_alignment(index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None):
|
||||
if species is not None:
|
||||
alignment = RegionAlignment(end - start, species, temp_file_handler=temp_file_handler)
|
||||
else:
|
||||
alignment = RegionAlignment(end - start, primary_species, temp_file_handler=temp_file_handler)
|
||||
return fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand, species, mincols, overwrite_with_gaps)
|
||||
|
||||
|
||||
# reduces a block to only positions exisiting in the src provided
|
||||
def reduce_block_by_primary_genome(block, species, chromosome, region_start):
|
||||
# returns ( startIndex, {species:texts}
|
||||
# where texts' contents are reduced to only positions existing in the primary genome
|
||||
src = "%s.%s" % (species, chromosome)
|
||||
ref = block.get_component_by_src(src)
|
||||
start_offset = ref.start - region_start
|
||||
species_texts = {}
|
||||
for c in block.components:
|
||||
species_texts[c.src.split('.')[0]] = list(c.text)
|
||||
# remove locations which are gaps in the primary species, starting from the downstream end
|
||||
for i in range(len(species_texts[species]) - 1, -1, -1):
|
||||
if species_texts[species][i] == '-':
|
||||
for text in species_texts.values():
|
||||
text.pop(i)
|
||||
for spec, text in species_texts.items():
|
||||
species_texts[spec] = ''.join(text)
|
||||
return (start_offset, species_texts)
|
||||
|
||||
|
||||
# fills a region alignment
|
||||
def fill_region_alignment(alignment, index, primary_species, chrom, start, end, strand='+', species=None, mincols=0, overwrite_with_gaps=True):
|
||||
region = bx.intervals.Interval(start, end)
|
||||
region.chrom = chrom
|
||||
region.strand = strand
|
||||
primary_src = "%s.%s" % (primary_species, chrom)
|
||||
|
||||
# Order blocks overlaping this position by score, lowest first
|
||||
blocks = []
|
||||
for block, idx, offset in index.get_as_iterator_with_index_and_offset(primary_src, start, end):
|
||||
score = float(block.score)
|
||||
for i in range(0, len(blocks)):
|
||||
if score < blocks[i][0]:
|
||||
blocks.insert(i, (score, idx, offset))
|
||||
break
|
||||
else:
|
||||
blocks.append((score, idx, offset))
|
||||
|
||||
# gap_chars_tuple = tuple( GAP_CHARS )
|
||||
gap_chars_str = ''.join(GAP_CHARS)
|
||||
# Loop through ordered blocks and layer by increasing score
|
||||
for block_dict in blocks:
|
||||
for block in iter_blocks_split_by_species(block_dict[1].get_at_offset(block_dict[2])): # need to handle each occurance of sequence in block seperately
|
||||
if component_overlaps_region(block.get_component_by_src(primary_src), region):
|
||||
block = chop_block_by_region(block, primary_src, region, species, mincols) # chop block
|
||||
block = orient_block_by_region(block, primary_src, region) # orient block
|
||||
start_offset, species_texts = reduce_block_by_primary_genome(block, primary_species, chrom, start)
|
||||
for spec, text in species_texts.items():
|
||||
# we should trim gaps from both sides, since these are not positions in this species genome (sequence)
|
||||
text = text.rstrip(gap_chars_str)
|
||||
gap_offset = 0
|
||||
# while text.startswith( gap_chars_tuple ):
|
||||
while True in [text.startswith(gap_char) for gap_char in GAP_CHARS]: # python2.4 doesn't accept a tuple for .startswith()
|
||||
gap_offset += 1
|
||||
text = text[1:]
|
||||
if not text:
|
||||
break
|
||||
if text:
|
||||
if overwrite_with_gaps:
|
||||
alignment.set_range(start_offset + gap_offset, spec, text)
|
||||
else:
|
||||
for i, char in enumerate(text):
|
||||
if char not in GAP_CHARS:
|
||||
alignment.set_position(start_offset + gap_offset + i, spec, char)
|
||||
return alignment
|
||||
|
||||
|
||||
# returns a filled spliced region alignment for specified region with start and end lists
|
||||
def get_spliced_region_alignment(index, primary_species, chrom, starts, ends, strand='+', species=None, mincols=0, overwrite_with_gaps=True, temp_file_handler=None):
|
||||
# create spliced alignment object
|
||||
if species is not None:
|
||||
alignment = SplicedAlignment(starts, ends, species, temp_file_handler=temp_file_handler)
|
||||
else:
|
||||
alignment = SplicedAlignment(starts, ends, [primary_species], temp_file_handler=temp_file_handler)
|
||||
for exon in alignment.exons:
|
||||
fill_region_alignment(exon, index, primary_species, chrom, exon.start, exon.end, strand, species, mincols, overwrite_with_gaps)
|
||||
return alignment
|
||||
|
||||
|
||||
# loop through string array, only return non-commented lines
|
||||
def line_enumerator(lines, comment_start='#'):
|
||||
i = 0
|
||||
for line in lines:
|
||||
if not line.startswith(comment_start):
|
||||
i += 1
|
||||
yield (i, line)
|
||||
|
||||
|
||||
# read a GeneBed file, return list of starts, ends, raw fields
|
||||
def get_starts_ends_fields_from_gene_bed(line):
|
||||
# Starts and ends for exons
|
||||
starts = []
|
||||
ends = []
|
||||
|
||||
fields = line.split()
|
||||
# Requires atleast 12 BED columns
|
||||
if len(fields) < 12:
|
||||
raise Exception("Not a proper 12 column BED line (%s)." % line)
|
||||
tx_start = int(fields[1])
|
||||
strand = fields[5]
|
||||
if strand != '-':
|
||||
strand = '+' # Default strand is +
|
||||
cds_start = int(fields[6])
|
||||
cds_end = int(fields[7])
|
||||
|
||||
# Calculate and store starts and ends of coding exons
|
||||
region_start, region_end = cds_start, cds_end
|
||||
exon_starts = list(map(int, fields[11].rstrip(',\n').split(',')))
|
||||
exon_starts = [x + tx_start for x in exon_starts]
|
||||
exon_ends = list(map(int, fields[10].rstrip(',').split(',')))
|
||||
exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)]
|
||||
for start, end in zip(exon_starts, exon_ends):
|
||||
start = max(start, region_start)
|
||||
end = min(end, region_end)
|
||||
if start < end:
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
return (starts, ends, fields)
|
||||
|
||||
|
||||
def iter_components_by_src(block, src):
|
||||
for c in block.components:
|
||||
if c.src == src:
|
||||
yield c
|
||||
|
||||
|
||||
def get_components_by_src(block, src):
|
||||
return [value for value in iter_components_by_src(block, src)]
|
||||
|
||||
|
||||
def iter_components_by_src_start(block, src):
|
||||
for c in block.components:
|
||||
if c.src.startswith(src):
|
||||
yield c
|
||||
|
||||
|
||||
def get_components_by_src_start(block, src):
|
||||
return [value for value in iter_components_by_src_start(block, src)]
|
||||
|
||||
|
||||
def sort_block_components_by_block(block1, block2):
|
||||
# orders the components in block1 by the index of the component in block2
|
||||
# block1 must be a subset of block2
|
||||
# occurs in-place
|
||||
return block1.components.sort(key=functools.cmp_to_key(lambda x, y: block2.components.index(x) - block2.components.index(y)))
|
||||
|
||||
|
||||
def get_species_in_maf(maf_filename):
|
||||
species = []
|
||||
for block in bx.align.maf.Reader(open(maf_filename)):
|
||||
for spec in get_species_in_block(block):
|
||||
if spec not in species:
|
||||
species.append(spec)
|
||||
return species
|
||||
|
||||
|
||||
def parse_species_option(species):
|
||||
if species:
|
||||
species = species.split(',')
|
||||
if 'None' not in species:
|
||||
return species
|
||||
return None # provided species was '', None, or had 'None' in it
|
||||
|
||||
|
||||
def remove_temp_index_file(index_filename):
|
||||
try:
|
||||
os.unlink(index_filename)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Below are methods to deal with FASTA files
|
||||
|
||||
|
||||
def get_fasta_header(component, attributes={}, suffix=None):
|
||||
header = ">%s(%s):%i-%i|" % (component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end())
|
||||
for key, value in attributes.items():
|
||||
header = "%s%s=%s|" % (header, key, value)
|
||||
if suffix:
|
||||
header = "%s%s" % (header, suffix)
|
||||
else:
|
||||
header = "%s%s" % (header, src_split(component.src)[0])
|
||||
return header
|
||||
|
||||
|
||||
def get_attributes_from_fasta_header(header):
|
||||
if not header:
|
||||
return {}
|
||||
attributes = {}
|
||||
header = header.lstrip('>')
|
||||
header = header.strip()
|
||||
fields = header.split('|')
|
||||
try:
|
||||
region = fields[0]
|
||||
region = region.split('(', 1)
|
||||
temp = region[0].split('.', 1)
|
||||
attributes['species'] = temp[0]
|
||||
if len(temp) == 2:
|
||||
attributes['chrom'] = temp[1]
|
||||
else:
|
||||
attributes['chrom'] = temp[0]
|
||||
region = region[1].split(')', 1)
|
||||
attributes['strand'] = region[0]
|
||||
region = region[1].lstrip(':').split('-')
|
||||
attributes['start'] = int(region[0])
|
||||
attributes['end'] = int(region[1])
|
||||
except Exception:
|
||||
# fields 0 is not a region coordinate
|
||||
pass
|
||||
if len(fields) > 2:
|
||||
for i in range(1, len(fields) - 1):
|
||||
prop = fields[i].split('=', 1)
|
||||
if len(prop) == 2:
|
||||
attributes[prop[0]] = prop[1]
|
||||
if len(fields) > 1:
|
||||
attributes['__suffix__'] = fields[-1]
|
||||
return attributes
|
||||
|
||||
|
||||
def iter_fasta_alignment(filename):
|
||||
class fastaComponent(object):
|
||||
def __init__(self, species, text=""):
|
||||
self.species = species
|
||||
self.text = text
|
||||
|
||||
def extend(self, text):
|
||||
self.text = self.text + text.replace('\n', '').replace('\r', '').strip()
|
||||
# yields a list of fastaComponents for a FASTA file
|
||||
f = open(filename, 'rb')
|
||||
components = []
|
||||
# cur_component = None
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
if components:
|
||||
yield components
|
||||
return
|
||||
line = line.strip()
|
||||
if not line:
|
||||
if components:
|
||||
yield components
|
||||
components = []
|
||||
elif line.startswith('>'):
|
||||
attributes = get_attributes_from_fasta_header(line)
|
||||
components.append(fastaComponent(attributes['species']))
|
||||
elif components:
|
||||
components[-1].extend(line)
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../datatypes/util/maf_utilities.py
|
||||
@@ -0,0 +1,377 @@
|
||||
import errno
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
from routes import url_for
|
||||
from six.moves.urllib.parse import urljoin
|
||||
|
||||
from galaxy import util
|
||||
from galaxy.util.odict import odict
|
||||
from galaxy.util.tool_shed import encoding_util, xml_util
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
REPOSITORY_OWNER = 'devteam'
|
||||
TOOL_MIGRATION_SCRIPTS_DIR = os.path.abspath(os.path.join(
|
||||
os.path.dirname(__file__), os.pardir, 'galaxy_install', 'migrate', 'scripts'))
|
||||
TOOL_MIGRATION_VERSIONS_DIR = os.path.abspath(os.path.join(
|
||||
os.path.dirname(__file__), os.pardir, 'galaxy_install', 'migrate', 'versions'))
|
||||
|
||||
|
||||
def accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies):
|
||||
if tool_shed_accessible:
|
||||
if tool_dependencies:
|
||||
for tool_dependency in tool_dependencies:
|
||||
if tool_dependency not in all_tool_dependencies:
|
||||
all_tool_dependencies.append(tool_dependency)
|
||||
return all_tool_dependencies
|
||||
|
||||
|
||||
def check_for_missing_tools(app, tool_panel_configs, latest_tool_migration_script_number):
|
||||
# Get the 000x_tools.xml file associated with the current migrate_tools version number.
|
||||
tools_xml_file_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
|
||||
os.pardir, 'galaxy_install',
|
||||
'migrate', 'scripts',
|
||||
'%04d_tools.xml' % latest_tool_migration_script_number))
|
||||
# Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
|
||||
migrated_tool_configs_dict = odict()
|
||||
tree, error_message = xml_util.parse_xml(tools_xml_file_path)
|
||||
if tree is None:
|
||||
return False, odict()
|
||||
root = tree.getroot()
|
||||
tool_shed = root.get('name')
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed)
|
||||
# The default behavior is that the tool shed is down.
|
||||
tool_shed_accessible = False
|
||||
missing_tool_configs_dict = odict()
|
||||
if tool_shed_url:
|
||||
for elem in root:
|
||||
if elem.tag == 'repository':
|
||||
repository_dependencies = []
|
||||
all_tool_dependencies = []
|
||||
repository_name = elem.get('name')
|
||||
changeset_revision = elem.get('changeset_revision')
|
||||
tool_shed_accessible, repository_dependencies_dict = get_repository_dependencies(app,
|
||||
tool_shed_url,
|
||||
repository_name,
|
||||
REPOSITORY_OWNER,
|
||||
changeset_revision)
|
||||
if tool_shed_accessible:
|
||||
# Accumulate all tool dependencies defined for repository dependencies for display to the user.
|
||||
for rd_key, rd_tups in repository_dependencies_dict.items():
|
||||
if rd_key in ['root_key', 'description']:
|
||||
continue
|
||||
for rd_tup in rd_tups:
|
||||
tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
|
||||
parse_repository_dependency_tuple(rd_tup)
|
||||
tool_shed_accessible, tool_dependencies = get_tool_dependencies(app,
|
||||
tool_shed_url,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision)
|
||||
all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies)
|
||||
tool_shed_accessible, tool_dependencies = get_tool_dependencies(app,
|
||||
tool_shed_url,
|
||||
repository_name,
|
||||
REPOSITORY_OWNER,
|
||||
changeset_revision)
|
||||
all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies)
|
||||
for tool_elem in elem.findall('tool'):
|
||||
tool_config_file_name = tool_elem.get('file')
|
||||
if tool_config_file_name:
|
||||
# We currently do nothing with repository dependencies except install them (we do not display repositories that will be
|
||||
# installed to the user). However, we'll store them in the following dictionary in case we choose to display them in the
|
||||
# future.
|
||||
dependencies_dict = dict(tool_dependencies=all_tool_dependencies,
|
||||
repository_dependencies=repository_dependencies)
|
||||
migrated_tool_configs_dict[tool_config_file_name] = dependencies_dict
|
||||
else:
|
||||
break
|
||||
if tool_shed_accessible:
|
||||
# Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names.
|
||||
for tool_panel_config in tool_panel_configs:
|
||||
tree, error_message = xml_util.parse_xml(tool_panel_config)
|
||||
if tree:
|
||||
root = tree.getroot()
|
||||
for elem in root:
|
||||
if elem.tag == 'tool':
|
||||
missing_tool_configs_dict = check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict)
|
||||
elif elem.tag == 'section':
|
||||
for section_elem in elem:
|
||||
if section_elem.tag == 'tool':
|
||||
missing_tool_configs_dict = check_tool_tag_set(section_elem, migrated_tool_configs_dict, missing_tool_configs_dict)
|
||||
else:
|
||||
exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % (tool_shed, app.config.tool_sheds_config)
|
||||
exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n'
|
||||
raise Exception(exception_msg)
|
||||
return tool_shed_accessible, missing_tool_configs_dict
|
||||
|
||||
|
||||
def check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict):
|
||||
file_path = elem.get('file', None)
|
||||
if file_path:
|
||||
name = os.path.basename(file_path)
|
||||
for migrated_tool_config in migrated_tool_configs_dict.keys():
|
||||
if migrated_tool_config in [file_path, name]:
|
||||
missing_tool_configs_dict[name] = migrated_tool_configs_dict[migrated_tool_config]
|
||||
return missing_tool_configs_dict
|
||||
|
||||
|
||||
def generate_clone_url_for_installed_repository(app, repository):
|
||||
"""Generate the URL for cloning a repository that has been installed into a Galaxy instance."""
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, str(repository.tool_shed))
|
||||
return util.build_url(tool_shed_url, pathspec=['repos', str(repository.owner), str(repository.name)])
|
||||
|
||||
|
||||
def generate_clone_url_for_repository_in_tool_shed(user, repository):
|
||||
"""Generate the URL for cloning a repository that is in the tool shed."""
|
||||
base_url = url_for('/', qualified=True).rstrip('/')
|
||||
if user:
|
||||
protocol, base = base_url.split('://')
|
||||
username = '%s@' % user.username
|
||||
return '%s://%s%s/repos/%s/%s' % (protocol, username, base, repository.user.username, repository.name)
|
||||
else:
|
||||
return '%s/repos/%s/%s' % (base_url, repository.user.username, repository.name)
|
||||
|
||||
|
||||
def generate_clone_url_from_repo_info_tup(app, repo_info_tup):
|
||||
"""Generate the URL for cloning a repository given a tuple of toolshed, name, owner, changeset_revision."""
|
||||
# Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab', False]
|
||||
toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
|
||||
parse_repository_dependency_tuple(repo_info_tup)
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, toolshed)
|
||||
# Don't include the changeset_revision in clone urls.
|
||||
return util.build_url(tool_shed_url, pathspec=['repos', owner, name])
|
||||
|
||||
|
||||
def get_non_shed_tool_panel_configs(app):
|
||||
"""Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml."""
|
||||
config_filenames = []
|
||||
for config_filename in app.config.tool_configs:
|
||||
# Any config file that includes a tool_path attribute in the root tag set like the following is shed-related.
|
||||
# <toolbox tool_path="database/shed_tools">
|
||||
try:
|
||||
tree, error_message = xml_util.parse_xml(config_filename)
|
||||
except (OSError, IOError) as exc:
|
||||
if (config_filename == app.config.shed_tool_conf and not
|
||||
app.config.shed_tool_conf_set and
|
||||
exc.errno == errno.ENOENT):
|
||||
continue
|
||||
raise
|
||||
if tree is None:
|
||||
continue
|
||||
root = tree.getroot()
|
||||
tool_path = root.get('tool_path', None)
|
||||
if tool_path is None:
|
||||
config_filenames.append(config_filename)
|
||||
return config_filenames
|
||||
|
||||
|
||||
def get_repository_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision):
|
||||
repository_dependencies_dict = {}
|
||||
tool_shed_accessible = True
|
||||
params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision)
|
||||
pathspec = ['repository', 'get_repository_dependencies']
|
||||
try:
|
||||
raw_text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params)
|
||||
tool_shed_accessible = True
|
||||
except Exception as e:
|
||||
tool_shed_accessible = False
|
||||
log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e)
|
||||
if tool_shed_accessible:
|
||||
if len(raw_text) > 2:
|
||||
encoded_text = json.loads(util.unicodify(raw_text))
|
||||
repository_dependencies_dict = encoding_util.tool_shed_decode(encoded_text)
|
||||
return tool_shed_accessible, repository_dependencies_dict
|
||||
|
||||
|
||||
def get_protocol_from_tool_shed_url(tool_shed_url):
|
||||
"""Return the protocol from the received tool_shed_url if it exists."""
|
||||
try:
|
||||
if tool_shed_url.find('://') > 0:
|
||||
return tool_shed_url.split('://')[0].lower()
|
||||
except Exception:
|
||||
# We receive a lot of calls here where the tool_shed_url is None. The container_util uses
|
||||
# that value when creating a header row. If the tool_shed_url is not None, we have a problem.
|
||||
if tool_shed_url is not None:
|
||||
log.exception("Handled exception getting the protocol from Tool Shed URL %s", str(tool_shed_url))
|
||||
# Default to HTTP protocol.
|
||||
return 'http'
|
||||
|
||||
|
||||
def get_tool_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision):
|
||||
tool_dependencies = []
|
||||
tool_shed_accessible = True
|
||||
params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision)
|
||||
pathspec = ['repository', 'get_tool_dependencies']
|
||||
try:
|
||||
text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params)
|
||||
tool_shed_accessible = True
|
||||
except Exception as e:
|
||||
tool_shed_accessible = False
|
||||
log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e)
|
||||
if tool_shed_accessible:
|
||||
if text:
|
||||
tool_dependencies_dict = encoding_util.tool_shed_decode(text)
|
||||
for requirements_dict in tool_dependencies_dict.values():
|
||||
tool_dependency_name = requirements_dict['name']
|
||||
tool_dependency_version = requirements_dict['version']
|
||||
tool_dependency_type = requirements_dict['type']
|
||||
tool_dependencies.append((tool_dependency_name, tool_dependency_version, tool_dependency_type))
|
||||
return tool_shed_accessible, tool_dependencies
|
||||
|
||||
|
||||
def get_tool_shed_repository_ids(as_string=False, **kwd):
|
||||
tsrid = kwd.get('tool_shed_repository_id', None)
|
||||
tsridslist = util.listify(kwd.get('tool_shed_repository_ids', None))
|
||||
if not tsridslist:
|
||||
tsridslist = util.listify(kwd.get('id', None))
|
||||
if tsridslist is not None:
|
||||
if tsrid is not None and tsrid not in tsridslist:
|
||||
tsridslist.append(tsrid)
|
||||
if as_string:
|
||||
return ','.join(tsridslist)
|
||||
return tsridslist
|
||||
else:
|
||||
tsridslist = util.listify(kwd.get('ordered_tsr_ids', None))
|
||||
if tsridslist is not None:
|
||||
if as_string:
|
||||
return ','.join(tsridslist)
|
||||
return tsridslist
|
||||
if as_string:
|
||||
return ''
|
||||
return []
|
||||
|
||||
|
||||
def get_tool_shed_url_from_tool_shed_registry(app, tool_shed):
|
||||
"""
|
||||
The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is
|
||||
something like: http://toolshed.g2.bx.psu.edu/
|
||||
"""
|
||||
cleaned_tool_shed = remove_protocol_from_tool_shed_url(tool_shed)
|
||||
for shed_url in app.tool_shed_registry.tool_sheds.values():
|
||||
if shed_url.find(cleaned_tool_shed) >= 0:
|
||||
if shed_url.endswith('/'):
|
||||
shed_url = shed_url.rstrip('/')
|
||||
return shed_url
|
||||
# The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml.
|
||||
return None
|
||||
|
||||
|
||||
def get_tool_shed_repository_url(app, tool_shed, owner, name):
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed)
|
||||
if tool_shed_url:
|
||||
# Append a slash to the tool shed URL, because urlparse.urljoin will eliminate
|
||||
# the last part of a URL if it does not end with a forward slash.
|
||||
tool_shed_url = '%s/' % tool_shed_url
|
||||
return urljoin(tool_shed_url, 'view/%s/%s' % (owner, name))
|
||||
return tool_shed_url
|
||||
|
||||
|
||||
def get_user_by_username(app, username):
|
||||
"""Get a user from the database by username."""
|
||||
sa_session = app.model.context.current
|
||||
try:
|
||||
user = sa_session.query(app.model.User) \
|
||||
.filter(app.model.User.table.c.username == username) \
|
||||
.one()
|
||||
return user
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def handle_galaxy_url(trans, **kwd):
|
||||
galaxy_url = kwd.get('galaxy_url', None)
|
||||
if galaxy_url:
|
||||
trans.set_cookie(galaxy_url, name='toolshedgalaxyurl')
|
||||
else:
|
||||
galaxy_url = trans.get_cookie(name='toolshedgalaxyurl')
|
||||
return galaxy_url
|
||||
|
||||
|
||||
def handle_tool_shed_url_protocol(app, shed_url):
|
||||
"""Handle secure and insecure HTTP protocol since they may change over time."""
|
||||
try:
|
||||
if app.name == 'galaxy':
|
||||
url = remove_protocol_from_tool_shed_url(shed_url)
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, url)
|
||||
else:
|
||||
tool_shed_url = str(url_for('/', qualified=True)).rstrip('/')
|
||||
return tool_shed_url
|
||||
except Exception:
|
||||
# We receive a lot of calls here where the tool_shed_url is None. The container_util uses
|
||||
# that value when creating a header row. If the tool_shed_url is not None, we have a problem.
|
||||
if shed_url is not None:
|
||||
log.exception("Handled exception removing protocol from URL %s", str(shed_url))
|
||||
return shed_url
|
||||
|
||||
|
||||
def parse_repository_dependency_tuple(repository_dependency_tuple, contains_error=False):
|
||||
# Default both prior_installation_required and only_if_compiling_contained_td to False in cases where metadata should be reset on the
|
||||
# repository containing the repository_dependency definition.
|
||||
prior_installation_required = 'False'
|
||||
only_if_compiling_contained_td = 'False'
|
||||
if contains_error:
|
||||
if len(repository_dependency_tuple) == 5:
|
||||
tool_shed, name, owner, changeset_revision, error = repository_dependency_tuple
|
||||
elif len(repository_dependency_tuple) == 6:
|
||||
tool_shed, name, owner, changeset_revision, prior_installation_required, error = repository_dependency_tuple
|
||||
elif len(repository_dependency_tuple) == 7:
|
||||
tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \
|
||||
repository_dependency_tuple
|
||||
return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error
|
||||
else:
|
||||
if len(repository_dependency_tuple) == 4:
|
||||
tool_shed, name, owner, changeset_revision = repository_dependency_tuple
|
||||
elif len(repository_dependency_tuple) == 5:
|
||||
tool_shed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tuple
|
||||
elif len(repository_dependency_tuple) == 6:
|
||||
tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tuple
|
||||
return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td
|
||||
|
||||
|
||||
def remove_port_from_tool_shed_url(tool_shed_url):
|
||||
"""Return a partial Tool Shed URL, eliminating the port if it exists."""
|
||||
try:
|
||||
if tool_shed_url.find(':') > 0:
|
||||
# Eliminate the port, if any, since it will result in an invalid directory name.
|
||||
new_tool_shed_url = tool_shed_url.split(':')[0]
|
||||
else:
|
||||
new_tool_shed_url = tool_shed_url
|
||||
return new_tool_shed_url.rstrip('/')
|
||||
except Exception:
|
||||
# We receive a lot of calls here where the tool_shed_url is None. The container_util uses
|
||||
# that value when creating a header row. If the tool_shed_url is not None, we have a problem.
|
||||
if tool_shed_url is not None:
|
||||
log.exception("Handled exception removing the port from Tool Shed URL %s", str(tool_shed_url))
|
||||
return tool_shed_url
|
||||
|
||||
|
||||
def remove_protocol_and_port_from_tool_shed_url(tool_shed_url):
|
||||
"""Return a partial Tool Shed URL, eliminating the protocol and/or port if either exists."""
|
||||
tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url)
|
||||
tool_shed = remove_port_from_tool_shed_url(tool_shed)
|
||||
return tool_shed
|
||||
|
||||
|
||||
def remove_protocol_and_user_from_clone_url(repository_clone_url):
|
||||
"""Return a URL that can be used to clone a repository, eliminating the protocol and user if either exists."""
|
||||
if repository_clone_url.find('@') > 0:
|
||||
# We have an url that includes an authenticated user, something like:
|
||||
# http://test@bx.psu.edu:9009/repos/some_username/column
|
||||
items = repository_clone_url.split('@')
|
||||
tmp_url = items[1]
|
||||
elif repository_clone_url.find('//') > 0:
|
||||
# We have an url that includes only a protocol, something like:
|
||||
# http://bx.psu.edu:9009/repos/some_username/column
|
||||
items = repository_clone_url.split('//')
|
||||
tmp_url = items[1]
|
||||
else:
|
||||
tmp_url = repository_clone_url
|
||||
return tmp_url.rstrip('/')
|
||||
|
||||
|
||||
def remove_protocol_from_tool_shed_url(tool_shed_url):
|
||||
"""Return a partial Tool Shed URL, eliminating the protocol if it exists."""
|
||||
return util.remove_protocol_from_url(tool_shed_url)
|
||||
@@ -0,0 +1,41 @@
|
||||
import binascii
|
||||
import json
|
||||
|
||||
from galaxy.util import (
|
||||
smart_str,
|
||||
unicodify
|
||||
)
|
||||
from galaxy.util.hash_util import hmac_new
|
||||
|
||||
|
||||
encoding_sep = '__esep__'
|
||||
encoding_sep2 = '__esepii__'
|
||||
|
||||
|
||||
def tool_shed_decode(value):
|
||||
# Extract and verify hash
|
||||
value = unicodify(value)
|
||||
a, b = value.split(":")
|
||||
value = binascii.unhexlify(b)
|
||||
test = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', value)
|
||||
assert a == test
|
||||
# Restore from string
|
||||
values = None
|
||||
value = unicodify(value)
|
||||
try:
|
||||
values = json.loads(value)
|
||||
except Exception:
|
||||
pass
|
||||
if values is None:
|
||||
values = value
|
||||
return values
|
||||
|
||||
|
||||
def tool_shed_encode(val):
|
||||
if isinstance(val, dict) or isinstance(val, list):
|
||||
value = json.dumps(val)
|
||||
else:
|
||||
value = val
|
||||
a = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', smart_str(value))
|
||||
b = unicodify(binascii.hexlify(smart_str(value)))
|
||||
return "%s:%s" % (a, b)
|
||||
@@ -0,0 +1,92 @@
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from xml.etree import ElementTree as XmlET
|
||||
|
||||
from galaxy.util import (
|
||||
xml_to_string
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Py27CommentedTreeBuilder(XmlET.TreeBuilder):
|
||||
|
||||
def doctype(*args):
|
||||
# handle deprecation warning for XMLParsing a file with DOCTYPE
|
||||
pass
|
||||
|
||||
def comment(self, data):
|
||||
self.start(XmlET.Comment, {})
|
||||
self.data(data)
|
||||
self.end(XmlET.Comment)
|
||||
|
||||
|
||||
def create_and_write_tmp_file(elem):
|
||||
tmp_str = xml_to_string(elem, pretty=True)
|
||||
with tempfile.NamedTemporaryFile(prefix="tmp-toolshed-cawrf", delete=False) as fh:
|
||||
tmp_filename = fh.name
|
||||
with io.open(tmp_filename, mode='w', encoding='utf-8') as fh:
|
||||
fh.write(tmp_str)
|
||||
return tmp_filename
|
||||
|
||||
|
||||
def create_element(tag, attributes=None, sub_elements=None):
|
||||
"""
|
||||
Create a new element whose tag is the value of the received tag, and whose attributes are all
|
||||
key / value pairs in the received attributes and sub_elements.
|
||||
"""
|
||||
if tag:
|
||||
elem = XmlET.Element(tag)
|
||||
if attributes:
|
||||
# The received attributes is an odict to preserve ordering.
|
||||
for k, v in attributes.items():
|
||||
elem.set(k, v)
|
||||
if sub_elements:
|
||||
# The received attributes is an odict. These handle information that tends to be
|
||||
# long text including paragraphs (e.g., description and long_description.
|
||||
for k, v in sub_elements.items():
|
||||
# Don't include fields that are blank.
|
||||
if v:
|
||||
if k == 'packages':
|
||||
# The received sub_elements is an odict whose key is 'packages' and whose
|
||||
# value is a list of ( name, version ) tuples.
|
||||
for v_tuple in v:
|
||||
sub_elem = XmlET.SubElement(elem, 'package')
|
||||
sub_elem_name, sub_elem_version = v_tuple
|
||||
sub_elem.set('name', sub_elem_name)
|
||||
sub_elem.set('version', sub_elem_version)
|
||||
elif isinstance(v, list):
|
||||
sub_elem = XmlET.SubElement(elem, k)
|
||||
# If v is a list, then it must be a list of tuples where the first
|
||||
# item is the tag and the second item is the text value.
|
||||
for v_tuple in v:
|
||||
if len(v_tuple) == 2:
|
||||
v_tag = v_tuple[0]
|
||||
v_text = v_tuple[1]
|
||||
# Don't include fields that are blank.
|
||||
if v_text:
|
||||
v_elem = XmlET.SubElement(sub_elem, v_tag)
|
||||
v_elem.text = v_text
|
||||
else:
|
||||
sub_elem = XmlET.SubElement(elem, k)
|
||||
sub_elem.text = v
|
||||
return elem
|
||||
return None
|
||||
|
||||
|
||||
def parse_xml(file_name):
|
||||
"""Returns a parsed xml tree with comments intact."""
|
||||
error_message = ''
|
||||
if not os.path.exists(file_name):
|
||||
return None, "File does not exist %s" % str(file_name)
|
||||
|
||||
with open(file_name, 'r') as fobj:
|
||||
try:
|
||||
tree = XmlET.parse(fobj, parser=XmlET.XMLParser(target=Py27CommentedTreeBuilder()))
|
||||
except Exception as e:
|
||||
error_message = "Exception attempting to parse %s: %s" % (str(file_name), str(e))
|
||||
log.exception(error_message)
|
||||
return None, error_message
|
||||
return tree, error_message
|
||||
@@ -1,377 +0,0 @@
|
||||
import errno
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
from six.moves.urllib.parse import urljoin
|
||||
|
||||
from galaxy import util
|
||||
from galaxy.util.odict import odict
|
||||
from galaxy.web import url_for
|
||||
from tool_shed.util import encoding_util, xml_util
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
REPOSITORY_OWNER = 'devteam'
|
||||
TOOL_MIGRATION_SCRIPTS_DIR = os.path.abspath(os.path.join(
|
||||
os.path.dirname(__file__), os.pardir, 'galaxy_install', 'migrate', 'scripts'))
|
||||
TOOL_MIGRATION_VERSIONS_DIR = os.path.abspath(os.path.join(
|
||||
os.path.dirname(__file__), os.pardir, 'galaxy_install', 'migrate', 'versions'))
|
||||
|
||||
|
||||
def accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies):
|
||||
if tool_shed_accessible:
|
||||
if tool_dependencies:
|
||||
for tool_dependency in tool_dependencies:
|
||||
if tool_dependency not in all_tool_dependencies:
|
||||
all_tool_dependencies.append(tool_dependency)
|
||||
return all_tool_dependencies
|
||||
|
||||
|
||||
def check_for_missing_tools(app, tool_panel_configs, latest_tool_migration_script_number):
|
||||
# Get the 000x_tools.xml file associated with the current migrate_tools version number.
|
||||
tools_xml_file_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
|
||||
os.pardir, 'galaxy_install',
|
||||
'migrate', 'scripts',
|
||||
'%04d_tools.xml' % latest_tool_migration_script_number))
|
||||
# Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
|
||||
migrated_tool_configs_dict = odict()
|
||||
tree, error_message = xml_util.parse_xml(tools_xml_file_path)
|
||||
if tree is None:
|
||||
return False, odict()
|
||||
root = tree.getroot()
|
||||
tool_shed = root.get('name')
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed)
|
||||
# The default behavior is that the tool shed is down.
|
||||
tool_shed_accessible = False
|
||||
missing_tool_configs_dict = odict()
|
||||
if tool_shed_url:
|
||||
for elem in root:
|
||||
if elem.tag == 'repository':
|
||||
repository_dependencies = []
|
||||
all_tool_dependencies = []
|
||||
repository_name = elem.get('name')
|
||||
changeset_revision = elem.get('changeset_revision')
|
||||
tool_shed_accessible, repository_dependencies_dict = get_repository_dependencies(app,
|
||||
tool_shed_url,
|
||||
repository_name,
|
||||
REPOSITORY_OWNER,
|
||||
changeset_revision)
|
||||
if tool_shed_accessible:
|
||||
# Accumulate all tool dependencies defined for repository dependencies for display to the user.
|
||||
for rd_key, rd_tups in repository_dependencies_dict.items():
|
||||
if rd_key in ['root_key', 'description']:
|
||||
continue
|
||||
for rd_tup in rd_tups:
|
||||
tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
|
||||
parse_repository_dependency_tuple(rd_tup)
|
||||
tool_shed_accessible, tool_dependencies = get_tool_dependencies(app,
|
||||
tool_shed_url,
|
||||
name,
|
||||
owner,
|
||||
changeset_revision)
|
||||
all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies)
|
||||
tool_shed_accessible, tool_dependencies = get_tool_dependencies(app,
|
||||
tool_shed_url,
|
||||
repository_name,
|
||||
REPOSITORY_OWNER,
|
||||
changeset_revision)
|
||||
all_tool_dependencies = accumulate_tool_dependencies(tool_shed_accessible, tool_dependencies, all_tool_dependencies)
|
||||
for tool_elem in elem.findall('tool'):
|
||||
tool_config_file_name = tool_elem.get('file')
|
||||
if tool_config_file_name:
|
||||
# We currently do nothing with repository dependencies except install them (we do not display repositories that will be
|
||||
# installed to the user). However, we'll store them in the following dictionary in case we choose to display them in the
|
||||
# future.
|
||||
dependencies_dict = dict(tool_dependencies=all_tool_dependencies,
|
||||
repository_dependencies=repository_dependencies)
|
||||
migrated_tool_configs_dict[tool_config_file_name] = dependencies_dict
|
||||
else:
|
||||
break
|
||||
if tool_shed_accessible:
|
||||
# Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names.
|
||||
for tool_panel_config in tool_panel_configs:
|
||||
tree, error_message = xml_util.parse_xml(tool_panel_config)
|
||||
if tree:
|
||||
root = tree.getroot()
|
||||
for elem in root:
|
||||
if elem.tag == 'tool':
|
||||
missing_tool_configs_dict = check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict)
|
||||
elif elem.tag == 'section':
|
||||
for section_elem in elem:
|
||||
if section_elem.tag == 'tool':
|
||||
missing_tool_configs_dict = check_tool_tag_set(section_elem, migrated_tool_configs_dict, missing_tool_configs_dict)
|
||||
else:
|
||||
exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % (tool_shed, app.config.tool_sheds_config)
|
||||
exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n'
|
||||
raise Exception(exception_msg)
|
||||
return tool_shed_accessible, missing_tool_configs_dict
|
||||
|
||||
|
||||
def check_tool_tag_set(elem, migrated_tool_configs_dict, missing_tool_configs_dict):
|
||||
file_path = elem.get('file', None)
|
||||
if file_path:
|
||||
name = os.path.basename(file_path)
|
||||
for migrated_tool_config in migrated_tool_configs_dict.keys():
|
||||
if migrated_tool_config in [file_path, name]:
|
||||
missing_tool_configs_dict[name] = migrated_tool_configs_dict[migrated_tool_config]
|
||||
return missing_tool_configs_dict
|
||||
|
||||
|
||||
def generate_clone_url_for_installed_repository(app, repository):
|
||||
"""Generate the URL for cloning a repository that has been installed into a Galaxy instance."""
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, str(repository.tool_shed))
|
||||
return util.build_url(tool_shed_url, pathspec=['repos', str(repository.owner), str(repository.name)])
|
||||
|
||||
|
||||
def generate_clone_url_for_repository_in_tool_shed(user, repository):
|
||||
"""Generate the URL for cloning a repository that is in the tool shed."""
|
||||
base_url = url_for('/', qualified=True).rstrip('/')
|
||||
if user:
|
||||
protocol, base = base_url.split('://')
|
||||
username = '%s@' % user.username
|
||||
return '%s://%s%s/repos/%s/%s' % (protocol, username, base, repository.user.username, repository.name)
|
||||
else:
|
||||
return '%s/repos/%s/%s' % (base_url, repository.user.username, repository.name)
|
||||
|
||||
|
||||
def generate_clone_url_from_repo_info_tup(app, repo_info_tup):
|
||||
"""Generate the URL for cloning a repository given a tuple of toolshed, name, owner, changeset_revision."""
|
||||
# Example tuple: ['http://localhost:9009', 'blast_datatypes', 'test', '461a4216e8ab', False]
|
||||
toolshed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
|
||||
parse_repository_dependency_tuple(repo_info_tup)
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, toolshed)
|
||||
# Don't include the changeset_revision in clone urls.
|
||||
return util.build_url(tool_shed_url, pathspec=['repos', owner, name])
|
||||
|
||||
|
||||
def get_non_shed_tool_panel_configs(app):
|
||||
"""Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml."""
|
||||
config_filenames = []
|
||||
for config_filename in app.config.tool_configs:
|
||||
# Any config file that includes a tool_path attribute in the root tag set like the following is shed-related.
|
||||
# <toolbox tool_path="database/shed_tools">
|
||||
try:
|
||||
tree, error_message = xml_util.parse_xml(config_filename)
|
||||
except (OSError, IOError) as exc:
|
||||
if (config_filename == app.config.shed_tool_conf and not
|
||||
app.config.shed_tool_conf_set and
|
||||
exc.errno == errno.ENOENT):
|
||||
continue
|
||||
raise
|
||||
if tree is None:
|
||||
continue
|
||||
root = tree.getroot()
|
||||
tool_path = root.get('tool_path', None)
|
||||
if tool_path is None:
|
||||
config_filenames.append(config_filename)
|
||||
return config_filenames
|
||||
|
||||
|
||||
def get_repository_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision):
|
||||
repository_dependencies_dict = {}
|
||||
tool_shed_accessible = True
|
||||
params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision)
|
||||
pathspec = ['repository', 'get_repository_dependencies']
|
||||
try:
|
||||
raw_text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params)
|
||||
tool_shed_accessible = True
|
||||
except Exception as e:
|
||||
tool_shed_accessible = False
|
||||
log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e)
|
||||
if tool_shed_accessible:
|
||||
if len(raw_text) > 2:
|
||||
encoded_text = json.loads(util.unicodify(raw_text))
|
||||
repository_dependencies_dict = encoding_util.tool_shed_decode(encoded_text)
|
||||
return tool_shed_accessible, repository_dependencies_dict
|
||||
|
||||
|
||||
def get_protocol_from_tool_shed_url(tool_shed_url):
|
||||
"""Return the protocol from the received tool_shed_url if it exists."""
|
||||
try:
|
||||
if tool_shed_url.find('://') > 0:
|
||||
return tool_shed_url.split('://')[0].lower()
|
||||
except Exception:
|
||||
# We receive a lot of calls here where the tool_shed_url is None. The container_util uses
|
||||
# that value when creating a header row. If the tool_shed_url is not None, we have a problem.
|
||||
if tool_shed_url is not None:
|
||||
log.exception("Handled exception getting the protocol from Tool Shed URL %s", str(tool_shed_url))
|
||||
# Default to HTTP protocol.
|
||||
return 'http'
|
||||
|
||||
|
||||
def get_tool_dependencies(app, tool_shed_url, repository_name, repository_owner, changeset_revision):
|
||||
tool_dependencies = []
|
||||
tool_shed_accessible = True
|
||||
params = dict(name=repository_name, owner=repository_owner, changeset_revision=changeset_revision)
|
||||
pathspec = ['repository', 'get_tool_dependencies']
|
||||
try:
|
||||
text = util.url_get(tool_shed_url, password_mgr=app.tool_shed_registry.url_auth(tool_shed_url), pathspec=pathspec, params=params)
|
||||
tool_shed_accessible = True
|
||||
except Exception as e:
|
||||
tool_shed_accessible = False
|
||||
log.warning("The URL\n%s\nraised the exception:\n%s\n", util.build_url(tool_shed_url, pathspec=pathspec, params=params), e)
|
||||
if tool_shed_accessible:
|
||||
if text:
|
||||
tool_dependencies_dict = encoding_util.tool_shed_decode(text)
|
||||
for requirements_dict in tool_dependencies_dict.values():
|
||||
tool_dependency_name = requirements_dict['name']
|
||||
tool_dependency_version = requirements_dict['version']
|
||||
tool_dependency_type = requirements_dict['type']
|
||||
tool_dependencies.append((tool_dependency_name, tool_dependency_version, tool_dependency_type))
|
||||
return tool_shed_accessible, tool_dependencies
|
||||
|
||||
|
||||
def get_tool_shed_repository_ids(as_string=False, **kwd):
|
||||
tsrid = kwd.get('tool_shed_repository_id', None)
|
||||
tsridslist = util.listify(kwd.get('tool_shed_repository_ids', None))
|
||||
if not tsridslist:
|
||||
tsridslist = util.listify(kwd.get('id', None))
|
||||
if tsridslist is not None:
|
||||
if tsrid is not None and tsrid not in tsridslist:
|
||||
tsridslist.append(tsrid)
|
||||
if as_string:
|
||||
return ','.join(tsridslist)
|
||||
return tsridslist
|
||||
else:
|
||||
tsridslist = util.listify(kwd.get('ordered_tsr_ids', None))
|
||||
if tsridslist is not None:
|
||||
if as_string:
|
||||
return ','.join(tsridslist)
|
||||
return tsridslist
|
||||
if as_string:
|
||||
return ''
|
||||
return []
|
||||
|
||||
|
||||
def get_tool_shed_url_from_tool_shed_registry(app, tool_shed):
|
||||
"""
|
||||
The value of tool_shed is something like: toolshed.g2.bx.psu.edu. We need the URL to this tool shed, which is
|
||||
something like: http://toolshed.g2.bx.psu.edu/
|
||||
"""
|
||||
cleaned_tool_shed = remove_protocol_from_tool_shed_url(tool_shed)
|
||||
for shed_url in app.tool_shed_registry.tool_sheds.values():
|
||||
if shed_url.find(cleaned_tool_shed) >= 0:
|
||||
if shed_url.endswith('/'):
|
||||
shed_url = shed_url.rstrip('/')
|
||||
return shed_url
|
||||
# The tool shed from which the repository was originally installed must no longer be configured in tool_sheds_conf.xml.
|
||||
return None
|
||||
|
||||
|
||||
def get_tool_shed_repository_url(app, tool_shed, owner, name):
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, tool_shed)
|
||||
if tool_shed_url:
|
||||
# Append a slash to the tool shed URL, because urlparse.urljoin will eliminate
|
||||
# the last part of a URL if it does not end with a forward slash.
|
||||
tool_shed_url = '%s/' % tool_shed_url
|
||||
return urljoin(tool_shed_url, 'view/%s/%s' % (owner, name))
|
||||
return tool_shed_url
|
||||
|
||||
|
||||
def get_user_by_username(app, username):
|
||||
"""Get a user from the database by username."""
|
||||
sa_session = app.model.context.current
|
||||
try:
|
||||
user = sa_session.query(app.model.User) \
|
||||
.filter(app.model.User.table.c.username == username) \
|
||||
.one()
|
||||
return user
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def handle_galaxy_url(trans, **kwd):
|
||||
galaxy_url = kwd.get('galaxy_url', None)
|
||||
if galaxy_url:
|
||||
trans.set_cookie(galaxy_url, name='toolshedgalaxyurl')
|
||||
else:
|
||||
galaxy_url = trans.get_cookie(name='toolshedgalaxyurl')
|
||||
return galaxy_url
|
||||
|
||||
|
||||
def handle_tool_shed_url_protocol(app, shed_url):
|
||||
"""Handle secure and insecure HTTP protocol since they may change over time."""
|
||||
try:
|
||||
if app.name == 'galaxy':
|
||||
url = remove_protocol_from_tool_shed_url(shed_url)
|
||||
tool_shed_url = get_tool_shed_url_from_tool_shed_registry(app, url)
|
||||
else:
|
||||
tool_shed_url = str(url_for('/', qualified=True)).rstrip('/')
|
||||
return tool_shed_url
|
||||
except Exception:
|
||||
# We receive a lot of calls here where the tool_shed_url is None. The container_util uses
|
||||
# that value when creating a header row. If the tool_shed_url is not None, we have a problem.
|
||||
if shed_url is not None:
|
||||
log.exception("Handled exception removing protocol from URL %s", str(shed_url))
|
||||
return shed_url
|
||||
|
||||
|
||||
def parse_repository_dependency_tuple(repository_dependency_tuple, contains_error=False):
|
||||
# Default both prior_installation_required and only_if_compiling_contained_td to False in cases where metadata should be reset on the
|
||||
# repository containing the repository_dependency definition.
|
||||
prior_installation_required = 'False'
|
||||
only_if_compiling_contained_td = 'False'
|
||||
if contains_error:
|
||||
if len(repository_dependency_tuple) == 5:
|
||||
tool_shed, name, owner, changeset_revision, error = repository_dependency_tuple
|
||||
elif len(repository_dependency_tuple) == 6:
|
||||
tool_shed, name, owner, changeset_revision, prior_installation_required, error = repository_dependency_tuple
|
||||
elif len(repository_dependency_tuple) == 7:
|
||||
tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error = \
|
||||
repository_dependency_tuple
|
||||
return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td, error
|
||||
else:
|
||||
if len(repository_dependency_tuple) == 4:
|
||||
tool_shed, name, owner, changeset_revision = repository_dependency_tuple
|
||||
elif len(repository_dependency_tuple) == 5:
|
||||
tool_shed, name, owner, changeset_revision, prior_installation_required = repository_dependency_tuple
|
||||
elif len(repository_dependency_tuple) == 6:
|
||||
tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = repository_dependency_tuple
|
||||
return tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td
|
||||
|
||||
|
||||
def remove_port_from_tool_shed_url(tool_shed_url):
|
||||
"""Return a partial Tool Shed URL, eliminating the port if it exists."""
|
||||
try:
|
||||
if tool_shed_url.find(':') > 0:
|
||||
# Eliminate the port, if any, since it will result in an invalid directory name.
|
||||
new_tool_shed_url = tool_shed_url.split(':')[0]
|
||||
else:
|
||||
new_tool_shed_url = tool_shed_url
|
||||
return new_tool_shed_url.rstrip('/')
|
||||
except Exception:
|
||||
# We receive a lot of calls here where the tool_shed_url is None. The container_util uses
|
||||
# that value when creating a header row. If the tool_shed_url is not None, we have a problem.
|
||||
if tool_shed_url is not None:
|
||||
log.exception("Handled exception removing the port from Tool Shed URL %s", str(tool_shed_url))
|
||||
return tool_shed_url
|
||||
|
||||
|
||||
def remove_protocol_and_port_from_tool_shed_url(tool_shed_url):
|
||||
"""Return a partial Tool Shed URL, eliminating the protocol and/or port if either exists."""
|
||||
tool_shed = remove_protocol_from_tool_shed_url(tool_shed_url)
|
||||
tool_shed = remove_port_from_tool_shed_url(tool_shed)
|
||||
return tool_shed
|
||||
|
||||
|
||||
def remove_protocol_and_user_from_clone_url(repository_clone_url):
|
||||
"""Return a URL that can be used to clone a repository, eliminating the protocol and user if either exists."""
|
||||
if repository_clone_url.find('@') > 0:
|
||||
# We have an url that includes an authenticated user, something like:
|
||||
# http://test@bx.psu.edu:9009/repos/some_username/column
|
||||
items = repository_clone_url.split('@')
|
||||
tmp_url = items[1]
|
||||
elif repository_clone_url.find('//') > 0:
|
||||
# We have an url that includes only a protocol, something like:
|
||||
# http://bx.psu.edu:9009/repos/some_username/column
|
||||
items = repository_clone_url.split('//')
|
||||
tmp_url = items[1]
|
||||
else:
|
||||
tmp_url = repository_clone_url
|
||||
return tmp_url.rstrip('/')
|
||||
|
||||
|
||||
def remove_protocol_from_tool_shed_url(tool_shed_url):
|
||||
"""Return a partial Tool Shed URL, eliminating the protocol if it exists."""
|
||||
return util.remove_protocol_from_url(tool_shed_url)
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../galaxy/util/tool_shed/common_util.py
|
||||
@@ -1,43 +0,0 @@
|
||||
import binascii
|
||||
import json
|
||||
import logging
|
||||
|
||||
from galaxy.util import (
|
||||
smart_str,
|
||||
unicodify
|
||||
)
|
||||
from galaxy.util.hash_util import hmac_new
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
encoding_sep = '__esep__'
|
||||
encoding_sep2 = '__esepii__'
|
||||
|
||||
|
||||
def tool_shed_decode(value):
|
||||
# Extract and verify hash
|
||||
value = unicodify(value)
|
||||
a, b = value.split(":")
|
||||
value = binascii.unhexlify(b)
|
||||
test = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', value)
|
||||
assert a == test
|
||||
# Restore from string
|
||||
values = None
|
||||
value = unicodify(value)
|
||||
try:
|
||||
values = json.loads(value)
|
||||
except Exception:
|
||||
pass
|
||||
if values is None:
|
||||
values = value
|
||||
return values
|
||||
|
||||
|
||||
def tool_shed_encode(val):
|
||||
if isinstance(val, dict) or isinstance(val, list):
|
||||
value = json.dumps(val)
|
||||
else:
|
||||
value = val
|
||||
a = hmac_new(b'ToolShedAndGalaxyMustHaveThisSameKey', smart_str(value))
|
||||
b = unicodify(binascii.hexlify(smart_str(value)))
|
||||
return "%s:%s" % (a, b)
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../galaxy/util/tool_shed/encoding_util.py
|
||||
@@ -1,92 +0,0 @@
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from xml.etree import ElementTree as XmlET
|
||||
|
||||
from galaxy.util import (
|
||||
xml_to_string
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Py27CommentedTreeBuilder(XmlET.TreeBuilder):
|
||||
|
||||
def doctype(*args):
|
||||
# handle deprecation warning for XMLParsing a file with DOCTYPE
|
||||
pass
|
||||
|
||||
def comment(self, data):
|
||||
self.start(XmlET.Comment, {})
|
||||
self.data(data)
|
||||
self.end(XmlET.Comment)
|
||||
|
||||
|
||||
def create_and_write_tmp_file(elem):
|
||||
tmp_str = xml_to_string(elem, pretty=True)
|
||||
with tempfile.NamedTemporaryFile(prefix="tmp-toolshed-cawrf", delete=False) as fh:
|
||||
tmp_filename = fh.name
|
||||
with io.open(tmp_filename, mode='w', encoding='utf-8') as fh:
|
||||
fh.write(tmp_str)
|
||||
return tmp_filename
|
||||
|
||||
|
||||
def create_element(tag, attributes=None, sub_elements=None):
|
||||
"""
|
||||
Create a new element whose tag is the value of the received tag, and whose attributes are all
|
||||
key / value pairs in the received attributes and sub_elements.
|
||||
"""
|
||||
if tag:
|
||||
elem = XmlET.Element(tag)
|
||||
if attributes:
|
||||
# The received attributes is an odict to preserve ordering.
|
||||
for k, v in attributes.items():
|
||||
elem.set(k, v)
|
||||
if sub_elements:
|
||||
# The received attributes is an odict. These handle information that tends to be
|
||||
# long text including paragraphs (e.g., description and long_description.
|
||||
for k, v in sub_elements.items():
|
||||
# Don't include fields that are blank.
|
||||
if v:
|
||||
if k == 'packages':
|
||||
# The received sub_elements is an odict whose key is 'packages' and whose
|
||||
# value is a list of ( name, version ) tuples.
|
||||
for v_tuple in v:
|
||||
sub_elem = XmlET.SubElement(elem, 'package')
|
||||
sub_elem_name, sub_elem_version = v_tuple
|
||||
sub_elem.set('name', sub_elem_name)
|
||||
sub_elem.set('version', sub_elem_version)
|
||||
elif isinstance(v, list):
|
||||
sub_elem = XmlET.SubElement(elem, k)
|
||||
# If v is a list, then it must be a list of tuples where the first
|
||||
# item is the tag and the second item is the text value.
|
||||
for v_tuple in v:
|
||||
if len(v_tuple) == 2:
|
||||
v_tag = v_tuple[0]
|
||||
v_text = v_tuple[1]
|
||||
# Don't include fields that are blank.
|
||||
if v_text:
|
||||
v_elem = XmlET.SubElement(sub_elem, v_tag)
|
||||
v_elem.text = v_text
|
||||
else:
|
||||
sub_elem = XmlET.SubElement(elem, k)
|
||||
sub_elem.text = v
|
||||
return elem
|
||||
return None
|
||||
|
||||
|
||||
def parse_xml(file_name):
|
||||
"""Returns a parsed xml tree with comments intact."""
|
||||
error_message = ''
|
||||
if not os.path.exists(file_name):
|
||||
return None, "File does not exist %s" % str(file_name)
|
||||
|
||||
with open(file_name, 'r') as fobj:
|
||||
try:
|
||||
tree = XmlET.parse(fobj, parser=XmlET.XMLParser(target=Py27CommentedTreeBuilder()))
|
||||
except Exception as e:
|
||||
error_message = "Exception attempting to parse %s: %s" % (str(file_name), str(e))
|
||||
log.exception(error_message)
|
||||
return None, error_message
|
||||
return tree, error_message
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../galaxy/util/tool_shed/xml_util.py
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../../lib/galaxy/app.py
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../../lib/galaxy/config.py
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../../lib/galaxy/config_watchers.py
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../../lib/galaxy/main.py
|
||||
+1
@@ -0,0 +1 @@
|
||||
../../../lib/galaxy/queue_worker.py
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../../lib/galaxy/queues.py
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../../lib/galaxy/version.py
|
||||
@@ -1,13 +1,16 @@
|
||||
galaxy-objectstore
|
||||
galaxy-util
|
||||
galaxy-sequence-utils==1.1.3
|
||||
galaxy-util[template]
|
||||
bdbag
|
||||
bx-python
|
||||
h5py
|
||||
isa-rwval
|
||||
parsley
|
||||
numpy<=1.16
|
||||
pycryptodome
|
||||
pysam
|
||||
social_auth_core
|
||||
SQLAlchemy==1.2
|
||||
sqlalchemy-migrate
|
||||
sqlalchemy-utils
|
||||
WebOb
|
||||
|
||||
@@ -1 +1 @@
|
||||
../../test/unit/unittest_utils/__init__.py
|
||||
../../../../test/unit/unittest_utils/__init__.py
|
||||
@@ -1 +1 @@
|
||||
../../test/unit/unittest_utils/tempfilecache.py
|
||||
../../../../test/unit/unittest_utils/tempfilecache.py
|
||||
@@ -1 +1 @@
|
||||
../../test/unit/unittest_utils/utility.py
|
||||
../../../../test/unit/unittest_utils/utility.py
|
||||
+1
-2
@@ -31,8 +31,7 @@ PACKAGE_DIRS=(
|
||||
)
|
||||
# containers has no tests, tool_util not yet working 100%,
|
||||
# data has many problems quota, tool shed install database, etc..
|
||||
RUN_TESTS=(1 1 1 0 1 0 1 0 1 0 0 0 0)
|
||||
|
||||
RUN_TESTS=(1 1 1 0 1 1 1 0 1 0 0 0 0)
|
||||
|
||||
for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do
|
||||
package_dir=${PACKAGE_DIRS[$i]}
|
||||
|
||||
@@ -5,4 +5,5 @@ docutils
|
||||
markupsafe
|
||||
packaging
|
||||
pyyaml
|
||||
routes
|
||||
six>=1.9.0
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ cat <<EOF
|
||||
'${0##*/} -list' for listing all the tool ids
|
||||
'${0##*/} -api (test_path)' for running all the test scripts in the ./test/api directory, test_path
|
||||
can be pytest selector
|
||||
'${0##*/} -integration (test_path)' for running all integration test scripts in the ./test/api directory, test_path
|
||||
'${0##*/} -integration (test_path)' for running all integration test scripts in the ./test/integration directory, test_path
|
||||
can be pytest selector
|
||||
'${0##*/} -toolshed (test_path)' for running all the test scripts in the ./test/shed_functional/functional directory
|
||||
'${0##*/} -installed' for running tests of Tool Shed installed tools
|
||||
|
||||
@@ -9,11 +9,11 @@ import routes
|
||||
from six import string_types
|
||||
|
||||
from galaxy import model
|
||||
from galaxy.config_watchers import ConfigWatchers
|
||||
from galaxy.model import tool_shed_install
|
||||
from galaxy.model.tool_shed_install import mapping
|
||||
from galaxy.tools import ToolBox
|
||||
from galaxy.tools.cache import ToolCache
|
||||
from galaxy.webapps.galaxy.config_watchers import ConfigWatchers
|
||||
from .test_tool_loader import (
|
||||
SIMPLE_MACRO,
|
||||
SIMPLE_TOOL_WITH_MACRO
|
||||
|
||||
Reference in New Issue
Block a user